diff --git a/weed/command/backup.go b/weed/command/backup.go index 86391f9c4..9c0bcbc52 100644 --- a/weed/command/backup.go +++ b/weed/command/backup.go @@ -88,7 +88,32 @@ func runBackup(cmd *Command, args []string) bool { return true } - if err := v.Synchronize(volumeServer, grpcDialOption); err != nil { + if v.SuperBlock.CompactRevision < uint16(stats.CompactRevision) { + if err = v.Compact(0); err != nil { + fmt.Printf("Compact Volume before synchronizing %v\n", err) + return true + } + if err = v.CommitCompact(); err != nil { + fmt.Printf("Commit Compact before synchronizing %v\n", err) + return true + } + v.SuperBlock.CompactRevision = uint16(stats.CompactRevision) + v.DataFile().WriteAt(v.SuperBlock.Bytes(),0) + } + + if uint64(v.Size()) > stats.TailOffset { + // remove the old data + v.Destroy() + // recreate an empty volume + v, err = storage.NewVolume(*s.dir, *s.collection, vid, storage.NeedleMapInMemory, replication, ttl, 0) + if err != nil { + fmt.Printf("Error creating or reading from volume %d: %v\n", vid, err) + return true + } + } + defer v.Close() + + if err := v.Follow(volumeServer, grpcDialOption); err != nil { fmt.Printf("Error synchronizing volume %d: %v\n", vid, err) return true } diff --git a/weed/pb/volume_server.proto b/weed/pb/volume_server.proto index 93db5b981..7686cc614 100644 --- a/weed/pb/volume_server.proto +++ b/weed/pb/volume_server.proto @@ -24,6 +24,8 @@ service VolumeServer { rpc VolumeSyncStatus (VolumeSyncStatusRequest) returns (VolumeSyncStatusResponse) { } + rpc VolumeFollow (VolumeFollowRequest) returns (stream VolumeFollowResponse) { + } rpc VolumeSyncIndex (VolumeSyncIndexRequest) returns (stream VolumeSyncIndexResponse) { } rpc VolumeSyncData (VolumeSyncDataRequest) returns (stream VolumeSyncDataResponse) { @@ -119,6 +121,14 @@ message VolumeSyncStatusResponse { uint64 idx_file_size = 8; } +message VolumeFollowRequest { + uint32 volume_id = 1; + uint64 since = 2; +} +message VolumeFollowResponse { + bytes file_content = 1; +} + message VolumeSyncIndexRequest { uint32 volume_id = 1; } diff --git a/weed/pb/volume_server_pb/volume_server.pb.go b/weed/pb/volume_server_pb/volume_server.pb.go index d84a5b099..4c5f1effa 100644 --- a/weed/pb/volume_server_pb/volume_server.pb.go +++ b/weed/pb/volume_server_pb/volume_server.pb.go @@ -27,6 +27,8 @@ It has these top-level messages: AssignVolumeResponse VolumeSyncStatusRequest VolumeSyncStatusResponse + VolumeFollowRequest + VolumeFollowResponse VolumeSyncIndexRequest VolumeSyncIndexResponse VolumeSyncDataRequest @@ -420,6 +422,46 @@ func (m *VolumeSyncStatusResponse) GetIdxFileSize() uint64 { return 0 } +type VolumeFollowRequest struct { + VolumeId uint32 `protobuf:"varint,1,opt,name=volume_id,json=volumeId" json:"volume_id,omitempty"` + Since uint64 `protobuf:"varint,2,opt,name=since" json:"since,omitempty"` +} + +func (m *VolumeFollowRequest) Reset() { *m = VolumeFollowRequest{} } +func (m *VolumeFollowRequest) String() string { return proto.CompactTextString(m) } +func (*VolumeFollowRequest) ProtoMessage() {} +func (*VolumeFollowRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{18} } + +func (m *VolumeFollowRequest) GetVolumeId() uint32 { + if m != nil { + return m.VolumeId + } + return 0 +} + +func (m *VolumeFollowRequest) GetSince() uint64 { + if m != nil { + return m.Since + } + return 0 +} + +type VolumeFollowResponse struct { + FileContent []byte `protobuf:"bytes,1,opt,name=file_content,json=fileContent,proto3" json:"file_content,omitempty"` +} + +func (m *VolumeFollowResponse) Reset() { *m = VolumeFollowResponse{} } +func (m *VolumeFollowResponse) String() string { return proto.CompactTextString(m) } +func (*VolumeFollowResponse) ProtoMessage() {} +func (*VolumeFollowResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{19} } + +func (m *VolumeFollowResponse) GetFileContent() []byte { + if m != nil { + return m.FileContent + } + return nil +} + type VolumeSyncIndexRequest struct { VolumeId uint32 `protobuf:"varint,1,opt,name=volume_id,json=volumeId" json:"volume_id,omitempty"` } @@ -427,7 +469,7 @@ type VolumeSyncIndexRequest struct { func (m *VolumeSyncIndexRequest) Reset() { *m = VolumeSyncIndexRequest{} } func (m *VolumeSyncIndexRequest) String() string { return proto.CompactTextString(m) } func (*VolumeSyncIndexRequest) ProtoMessage() {} -func (*VolumeSyncIndexRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{18} } +func (*VolumeSyncIndexRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{20} } func (m *VolumeSyncIndexRequest) GetVolumeId() uint32 { if m != nil { @@ -443,7 +485,7 @@ type VolumeSyncIndexResponse struct { func (m *VolumeSyncIndexResponse) Reset() { *m = VolumeSyncIndexResponse{} } func (m *VolumeSyncIndexResponse) String() string { return proto.CompactTextString(m) } func (*VolumeSyncIndexResponse) ProtoMessage() {} -func (*VolumeSyncIndexResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{19} } +func (*VolumeSyncIndexResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{21} } func (m *VolumeSyncIndexResponse) GetIndexFileContent() []byte { if m != nil { @@ -463,7 +505,7 @@ type VolumeSyncDataRequest struct { func (m *VolumeSyncDataRequest) Reset() { *m = VolumeSyncDataRequest{} } func (m *VolumeSyncDataRequest) String() string { return proto.CompactTextString(m) } func (*VolumeSyncDataRequest) ProtoMessage() {} -func (*VolumeSyncDataRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{20} } +func (*VolumeSyncDataRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{22} } func (m *VolumeSyncDataRequest) GetVolumeId() uint32 { if m != nil { @@ -507,7 +549,7 @@ type VolumeSyncDataResponse struct { func (m *VolumeSyncDataResponse) Reset() { *m = VolumeSyncDataResponse{} } func (m *VolumeSyncDataResponse) String() string { return proto.CompactTextString(m) } func (*VolumeSyncDataResponse) ProtoMessage() {} -func (*VolumeSyncDataResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{21} } +func (*VolumeSyncDataResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{23} } func (m *VolumeSyncDataResponse) GetFileContent() []byte { if m != nil { @@ -523,7 +565,7 @@ type VolumeMountRequest struct { func (m *VolumeMountRequest) Reset() { *m = VolumeMountRequest{} } func (m *VolumeMountRequest) String() string { return proto.CompactTextString(m) } func (*VolumeMountRequest) ProtoMessage() {} -func (*VolumeMountRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{22} } +func (*VolumeMountRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{24} } func (m *VolumeMountRequest) GetVolumeId() uint32 { if m != nil { @@ -538,7 +580,7 @@ type VolumeMountResponse struct { func (m *VolumeMountResponse) Reset() { *m = VolumeMountResponse{} } func (m *VolumeMountResponse) String() string { return proto.CompactTextString(m) } func (*VolumeMountResponse) ProtoMessage() {} -func (*VolumeMountResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{23} } +func (*VolumeMountResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{25} } type VolumeUnmountRequest struct { VolumeId uint32 `protobuf:"varint,1,opt,name=volume_id,json=volumeId" json:"volume_id,omitempty"` @@ -547,7 +589,7 @@ type VolumeUnmountRequest struct { func (m *VolumeUnmountRequest) Reset() { *m = VolumeUnmountRequest{} } func (m *VolumeUnmountRequest) String() string { return proto.CompactTextString(m) } func (*VolumeUnmountRequest) ProtoMessage() {} -func (*VolumeUnmountRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{24} } +func (*VolumeUnmountRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{26} } func (m *VolumeUnmountRequest) GetVolumeId() uint32 { if m != nil { @@ -562,7 +604,7 @@ type VolumeUnmountResponse struct { func (m *VolumeUnmountResponse) Reset() { *m = VolumeUnmountResponse{} } func (m *VolumeUnmountResponse) String() string { return proto.CompactTextString(m) } func (*VolumeUnmountResponse) ProtoMessage() {} -func (*VolumeUnmountResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{25} } +func (*VolumeUnmountResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{27} } type VolumeDeleteRequest struct { VolumeId uint32 `protobuf:"varint,1,opt,name=volume_id,json=volumeId" json:"volume_id,omitempty"` @@ -571,7 +613,7 @@ type VolumeDeleteRequest struct { func (m *VolumeDeleteRequest) Reset() { *m = VolumeDeleteRequest{} } func (m *VolumeDeleteRequest) String() string { return proto.CompactTextString(m) } func (*VolumeDeleteRequest) ProtoMessage() {} -func (*VolumeDeleteRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{26} } +func (*VolumeDeleteRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{28} } func (m *VolumeDeleteRequest) GetVolumeId() uint32 { if m != nil { @@ -586,7 +628,7 @@ type VolumeDeleteResponse struct { func (m *VolumeDeleteResponse) Reset() { *m = VolumeDeleteResponse{} } func (m *VolumeDeleteResponse) String() string { return proto.CompactTextString(m) } func (*VolumeDeleteResponse) ProtoMessage() {} -func (*VolumeDeleteResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{27} } +func (*VolumeDeleteResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{29} } type ReplicateVolumeRequest struct { VolumeId uint32 `protobuf:"varint,1,opt,name=volume_id,json=volumeId" json:"volume_id,omitempty"` @@ -599,7 +641,7 @@ type ReplicateVolumeRequest struct { func (m *ReplicateVolumeRequest) Reset() { *m = ReplicateVolumeRequest{} } func (m *ReplicateVolumeRequest) String() string { return proto.CompactTextString(m) } func (*ReplicateVolumeRequest) ProtoMessage() {} -func (*ReplicateVolumeRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{28} } +func (*ReplicateVolumeRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{30} } func (m *ReplicateVolumeRequest) GetVolumeId() uint32 { if m != nil { @@ -642,7 +684,7 @@ type ReplicateVolumeResponse struct { func (m *ReplicateVolumeResponse) Reset() { *m = ReplicateVolumeResponse{} } func (m *ReplicateVolumeResponse) String() string { return proto.CompactTextString(m) } func (*ReplicateVolumeResponse) ProtoMessage() {} -func (*ReplicateVolumeResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{29} } +func (*ReplicateVolumeResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{31} } type CopyFileRequest struct { VolumeId uint32 `protobuf:"varint,1,opt,name=volume_id,json=volumeId" json:"volume_id,omitempty"` @@ -653,7 +695,7 @@ type CopyFileRequest struct { func (m *CopyFileRequest) Reset() { *m = CopyFileRequest{} } func (m *CopyFileRequest) String() string { return proto.CompactTextString(m) } func (*CopyFileRequest) ProtoMessage() {} -func (*CopyFileRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{30} } +func (*CopyFileRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{32} } func (m *CopyFileRequest) GetVolumeId() uint32 { if m != nil { @@ -683,7 +725,7 @@ type CopyFileResponse struct { func (m *CopyFileResponse) Reset() { *m = CopyFileResponse{} } func (m *CopyFileResponse) String() string { return proto.CompactTextString(m) } func (*CopyFileResponse) ProtoMessage() {} -func (*CopyFileResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{31} } +func (*CopyFileResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{33} } func (m *CopyFileResponse) GetFileContent() []byte { if m != nil { @@ -699,7 +741,7 @@ type ReadVolumeFileStatusRequest struct { func (m *ReadVolumeFileStatusRequest) Reset() { *m = ReadVolumeFileStatusRequest{} } func (m *ReadVolumeFileStatusRequest) String() string { return proto.CompactTextString(m) } func (*ReadVolumeFileStatusRequest) ProtoMessage() {} -func (*ReadVolumeFileStatusRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{32} } +func (*ReadVolumeFileStatusRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{34} } func (m *ReadVolumeFileStatusRequest) GetVolumeId() uint32 { if m != nil { @@ -719,7 +761,7 @@ type ReadVolumeFileStatusResponse struct { func (m *ReadVolumeFileStatusResponse) Reset() { *m = ReadVolumeFileStatusResponse{} } func (m *ReadVolumeFileStatusResponse) String() string { return proto.CompactTextString(m) } func (*ReadVolumeFileStatusResponse) ProtoMessage() {} -func (*ReadVolumeFileStatusResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{33} } +func (*ReadVolumeFileStatusResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{35} } func (m *ReadVolumeFileStatusResponse) GetVolumeId() uint32 { if m != nil { @@ -766,7 +808,7 @@ type DiskStatus struct { func (m *DiskStatus) Reset() { *m = DiskStatus{} } func (m *DiskStatus) String() string { return proto.CompactTextString(m) } func (*DiskStatus) ProtoMessage() {} -func (*DiskStatus) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{34} } +func (*DiskStatus) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{36} } func (m *DiskStatus) GetDir() string { if m != nil { @@ -809,7 +851,7 @@ type MemStatus struct { func (m *MemStatus) Reset() { *m = MemStatus{} } func (m *MemStatus) String() string { return proto.CompactTextString(m) } func (*MemStatus) ProtoMessage() {} -func (*MemStatus) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{35} } +func (*MemStatus) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{37} } func (m *MemStatus) GetGoroutines() int32 { if m != nil { @@ -879,6 +921,8 @@ func init() { proto.RegisterType((*AssignVolumeResponse)(nil), "volume_server_pb.AssignVolumeResponse") proto.RegisterType((*VolumeSyncStatusRequest)(nil), "volume_server_pb.VolumeSyncStatusRequest") proto.RegisterType((*VolumeSyncStatusResponse)(nil), "volume_server_pb.VolumeSyncStatusResponse") + proto.RegisterType((*VolumeFollowRequest)(nil), "volume_server_pb.VolumeFollowRequest") + proto.RegisterType((*VolumeFollowResponse)(nil), "volume_server_pb.VolumeFollowResponse") proto.RegisterType((*VolumeSyncIndexRequest)(nil), "volume_server_pb.VolumeSyncIndexRequest") proto.RegisterType((*VolumeSyncIndexResponse)(nil), "volume_server_pb.VolumeSyncIndexResponse") proto.RegisterType((*VolumeSyncDataRequest)(nil), "volume_server_pb.VolumeSyncDataRequest") @@ -919,6 +963,7 @@ type VolumeServerClient interface { DeleteCollection(ctx context.Context, in *DeleteCollectionRequest, opts ...grpc.CallOption) (*DeleteCollectionResponse, error) AssignVolume(ctx context.Context, in *AssignVolumeRequest, opts ...grpc.CallOption) (*AssignVolumeResponse, error) VolumeSyncStatus(ctx context.Context, in *VolumeSyncStatusRequest, opts ...grpc.CallOption) (*VolumeSyncStatusResponse, error) + VolumeFollow(ctx context.Context, in *VolumeFollowRequest, opts ...grpc.CallOption) (VolumeServer_VolumeFollowClient, error) VolumeSyncIndex(ctx context.Context, in *VolumeSyncIndexRequest, opts ...grpc.CallOption) (VolumeServer_VolumeSyncIndexClient, error) VolumeSyncData(ctx context.Context, in *VolumeSyncDataRequest, opts ...grpc.CallOption) (VolumeServer_VolumeSyncDataClient, error) VolumeMount(ctx context.Context, in *VolumeMountRequest, opts ...grpc.CallOption) (*VolumeMountResponse, error) @@ -1009,8 +1054,40 @@ func (c *volumeServerClient) VolumeSyncStatus(ctx context.Context, in *VolumeSyn return out, nil } +func (c *volumeServerClient) VolumeFollow(ctx context.Context, in *VolumeFollowRequest, opts ...grpc.CallOption) (VolumeServer_VolumeFollowClient, error) { + stream, err := grpc.NewClientStream(ctx, &_VolumeServer_serviceDesc.Streams[0], c.cc, "/volume_server_pb.VolumeServer/VolumeFollow", opts...) + if err != nil { + return nil, err + } + x := &volumeServerVolumeFollowClient{stream} + if err := x.ClientStream.SendMsg(in); err != nil { + return nil, err + } + if err := x.ClientStream.CloseSend(); err != nil { + return nil, err + } + return x, nil +} + +type VolumeServer_VolumeFollowClient interface { + Recv() (*VolumeFollowResponse, error) + grpc.ClientStream +} + +type volumeServerVolumeFollowClient struct { + grpc.ClientStream +} + +func (x *volumeServerVolumeFollowClient) Recv() (*VolumeFollowResponse, error) { + m := new(VolumeFollowResponse) + if err := x.ClientStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + func (c *volumeServerClient) VolumeSyncIndex(ctx context.Context, in *VolumeSyncIndexRequest, opts ...grpc.CallOption) (VolumeServer_VolumeSyncIndexClient, error) { - stream, err := grpc.NewClientStream(ctx, &_VolumeServer_serviceDesc.Streams[0], c.cc, "/volume_server_pb.VolumeServer/VolumeSyncIndex", opts...) + stream, err := grpc.NewClientStream(ctx, &_VolumeServer_serviceDesc.Streams[1], c.cc, "/volume_server_pb.VolumeServer/VolumeSyncIndex", opts...) if err != nil { return nil, err } @@ -1042,7 +1119,7 @@ func (x *volumeServerVolumeSyncIndexClient) Recv() (*VolumeSyncIndexResponse, er } func (c *volumeServerClient) VolumeSyncData(ctx context.Context, in *VolumeSyncDataRequest, opts ...grpc.CallOption) (VolumeServer_VolumeSyncDataClient, error) { - stream, err := grpc.NewClientStream(ctx, &_VolumeServer_serviceDesc.Streams[1], c.cc, "/volume_server_pb.VolumeServer/VolumeSyncData", opts...) + stream, err := grpc.NewClientStream(ctx, &_VolumeServer_serviceDesc.Streams[2], c.cc, "/volume_server_pb.VolumeServer/VolumeSyncData", opts...) if err != nil { return nil, err } @@ -1119,7 +1196,7 @@ func (c *volumeServerClient) ReadVolumeFileStatus(ctx context.Context, in *ReadV } func (c *volumeServerClient) CopyFile(ctx context.Context, in *CopyFileRequest, opts ...grpc.CallOption) (VolumeServer_CopyFileClient, error) { - stream, err := grpc.NewClientStream(ctx, &_VolumeServer_serviceDesc.Streams[2], c.cc, "/volume_server_pb.VolumeServer/CopyFile", opts...) + stream, err := grpc.NewClientStream(ctx, &_VolumeServer_serviceDesc.Streams[3], c.cc, "/volume_server_pb.VolumeServer/CopyFile", opts...) if err != nil { return nil, err } @@ -1162,6 +1239,7 @@ type VolumeServerServer interface { DeleteCollection(context.Context, *DeleteCollectionRequest) (*DeleteCollectionResponse, error) AssignVolume(context.Context, *AssignVolumeRequest) (*AssignVolumeResponse, error) VolumeSyncStatus(context.Context, *VolumeSyncStatusRequest) (*VolumeSyncStatusResponse, error) + VolumeFollow(*VolumeFollowRequest, VolumeServer_VolumeFollowServer) error VolumeSyncIndex(*VolumeSyncIndexRequest, VolumeServer_VolumeSyncIndexServer) error VolumeSyncData(*VolumeSyncDataRequest, VolumeServer_VolumeSyncDataServer) error VolumeMount(context.Context, *VolumeMountRequest) (*VolumeMountResponse, error) @@ -1320,6 +1398,27 @@ func _VolumeServer_VolumeSyncStatus_Handler(srv interface{}, ctx context.Context return interceptor(ctx, in, info, handler) } +func _VolumeServer_VolumeFollow_Handler(srv interface{}, stream grpc.ServerStream) error { + m := new(VolumeFollowRequest) + if err := stream.RecvMsg(m); err != nil { + return err + } + return srv.(VolumeServerServer).VolumeFollow(m, &volumeServerVolumeFollowServer{stream}) +} + +type VolumeServer_VolumeFollowServer interface { + Send(*VolumeFollowResponse) error + grpc.ServerStream +} + +type volumeServerVolumeFollowServer struct { + grpc.ServerStream +} + +func (x *volumeServerVolumeFollowServer) Send(m *VolumeFollowResponse) error { + return x.ServerStream.SendMsg(m) +} + func _VolumeServer_VolumeSyncIndex_Handler(srv interface{}, stream grpc.ServerStream) error { m := new(VolumeSyncIndexRequest) if err := stream.RecvMsg(m); err != nil { @@ -1531,6 +1630,11 @@ var _VolumeServer_serviceDesc = grpc.ServiceDesc{ }, }, Streams: []grpc.StreamDesc{ + { + StreamName: "VolumeFollow", + Handler: _VolumeServer_VolumeFollow_Handler, + ServerStreams: true, + }, { StreamName: "VolumeSyncIndex", Handler: _VolumeServer_VolumeSyncIndex_Handler, @@ -1553,83 +1657,86 @@ var _VolumeServer_serviceDesc = grpc.ServiceDesc{ func init() { proto.RegisterFile("volume_server.proto", fileDescriptor0) } var fileDescriptor0 = []byte{ - // 1247 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xac, 0x58, 0x5b, 0x73, 0xdb, 0xc4, - 0x17, 0x8f, 0x62, 0x3b, 0x71, 0x8e, 0xed, 0xc6, 0xff, 0x4d, 0x9a, 0x38, 0x4a, 0xff, 0xc1, 0x15, - 0x90, 0x3a, 0x6d, 0x1a, 0x20, 0x9d, 0x42, 0x81, 0x17, 0x20, 0x01, 0x26, 0x0f, 0xa5, 0x33, 0x0a, - 0xed, 0x30, 0x43, 0x67, 0x34, 0x1b, 0x69, 0x9d, 0x88, 0xc8, 0x92, 0xaa, 0x5d, 0x85, 0x84, 0x6f, - 0xc2, 0x33, 0x2f, 0x7d, 0xe7, 0x03, 0xf1, 0x41, 0x78, 0x61, 0xf6, 0x22, 0x59, 0x37, 0xc7, 0xe2, - 0xf2, 0xb6, 0x3a, 0x7b, 0xce, 0xef, 0x5c, 0xf6, 0xec, 0xd9, 0x9f, 0x0d, 0x6b, 0x57, 0x81, 0x17, - 0x4f, 0x88, 0x45, 0x49, 0x74, 0x45, 0xa2, 0x83, 0x30, 0x0a, 0x58, 0x80, 0xfa, 0x39, 0xa1, 0x15, - 0x9e, 0x19, 0x1f, 0x00, 0xfa, 0x0a, 0x33, 0xfb, 0xe2, 0x98, 0x78, 0x84, 0x11, 0x93, 0xbc, 0x89, - 0x09, 0x65, 0x68, 0x0b, 0xda, 0x63, 0xd7, 0x23, 0x96, 0xeb, 0xd0, 0x81, 0x36, 0x6c, 0x8c, 0x56, - 0xcc, 0x65, 0xfe, 0x7d, 0xe2, 0x50, 0xe3, 0x05, 0xac, 0xe5, 0x0c, 0x68, 0x18, 0xf8, 0x94, 0xa0, - 0x67, 0xb0, 0x1c, 0x11, 0x1a, 0x7b, 0x4c, 0x1a, 0x74, 0x0e, 0x77, 0x0e, 0x8a, 0xbe, 0x0e, 0x52, - 0x93, 0xd8, 0x63, 0x66, 0xa2, 0x6e, 0xb8, 0xd0, 0xcd, 0x6e, 0xa0, 0x4d, 0x58, 0x56, 0xbe, 0x07, - 0xda, 0x50, 0x1b, 0xad, 0x98, 0x4b, 0xd2, 0x35, 0xda, 0x80, 0x25, 0xca, 0x30, 0x8b, 0xe9, 0x60, - 0x71, 0xa8, 0x8d, 0x5a, 0xa6, 0xfa, 0x42, 0xeb, 0xd0, 0x22, 0x51, 0x14, 0x44, 0x83, 0x86, 0x50, - 0x97, 0x1f, 0x08, 0x41, 0x93, 0xba, 0xbf, 0x90, 0x41, 0x73, 0xa8, 0x8d, 0x7a, 0xa6, 0x58, 0x1b, - 0xcb, 0xd0, 0xfa, 0x7a, 0x12, 0xb2, 0x1b, 0xe3, 0x13, 0x18, 0xbc, 0xc2, 0x76, 0x1c, 0x4f, 0x5e, - 0x89, 0x18, 0x8f, 0x2e, 0x88, 0x7d, 0x99, 0xe4, 0xbe, 0x0d, 0x2b, 0x2a, 0x72, 0x15, 0x41, 0xcf, - 0x6c, 0x4b, 0xc1, 0x89, 0x63, 0x7c, 0x01, 0x5b, 0x15, 0x86, 0xaa, 0x06, 0xef, 0x42, 0xef, 0x1c, - 0x47, 0x67, 0xf8, 0x9c, 0x58, 0x11, 0x66, 0x6e, 0x20, 0xac, 0x35, 0xb3, 0xab, 0x84, 0x26, 0x97, - 0x19, 0x3f, 0x82, 0x9e, 0x43, 0x08, 0x26, 0x21, 0xb6, 0x59, 0x1d, 0xe7, 0x68, 0x08, 0x9d, 0x30, - 0x22, 0xd8, 0xf3, 0x02, 0x1b, 0x33, 0x22, 0xaa, 0xd0, 0x30, 0xb3, 0x22, 0xe3, 0xff, 0xb0, 0x5d, - 0x09, 0x2e, 0x03, 0x34, 0x9e, 0x15, 0xa2, 0x0f, 0x26, 0x13, 0xb7, 0x96, 0x6b, 0xe3, 0x5e, 0x29, - 0x6a, 0x61, 0xa9, 0x70, 0x3f, 0x2d, 0xec, 0x7a, 0x04, 0xfb, 0x71, 0x58, 0x0b, 0xb8, 0x18, 0x71, - 0x62, 0x9a, 0x22, 0x6f, 0xca, 0xe6, 0x38, 0x0a, 0x3c, 0x8f, 0xd8, 0xcc, 0x0d, 0xfc, 0x04, 0x76, - 0x07, 0xc0, 0x4e, 0x85, 0xaa, 0x55, 0x32, 0x12, 0x43, 0x87, 0x41, 0xd9, 0x54, 0xc1, 0xbe, 0xd5, - 0x60, 0xed, 0x4b, 0x4a, 0xdd, 0x73, 0x5f, 0xba, 0xad, 0x55, 0xfe, 0xbc, 0xc3, 0xc5, 0xa2, 0xc3, - 0xe2, 0xf1, 0x34, 0x4a, 0xc7, 0xc3, 0x35, 0x22, 0x12, 0x7a, 0xae, 0x8d, 0x05, 0x44, 0x53, 0x40, - 0x64, 0x45, 0xa8, 0x0f, 0x0d, 0xc6, 0xbc, 0x41, 0x4b, 0xec, 0xf0, 0xa5, 0xb1, 0x01, 0xeb, 0xf9, - 0x48, 0x55, 0x0a, 0x1f, 0xc3, 0xa6, 0x94, 0x9c, 0xde, 0xf8, 0xf6, 0xa9, 0xb8, 0x09, 0xb5, 0x0a, - 0xfe, 0xa7, 0x06, 0x83, 0xb2, 0xa1, 0xea, 0xe0, 0x7f, 0x9b, 0xff, 0xdf, 0xcd, 0x0e, 0xbd, 0x03, - 0x1d, 0x86, 0x5d, 0xcf, 0x0a, 0xc6, 0x63, 0x4a, 0xd8, 0x60, 0x69, 0xa8, 0x8d, 0x9a, 0x26, 0x70, - 0xd1, 0x0b, 0x21, 0x41, 0x7b, 0xd0, 0xb7, 0x65, 0x17, 0x5b, 0x11, 0xb9, 0x72, 0x29, 0x47, 0x5e, - 0x16, 0x81, 0xad, 0xda, 0x49, 0x77, 0x4b, 0x31, 0x32, 0xa0, 0xe7, 0x3a, 0xd7, 0x96, 0x18, 0x1e, - 0xe2, 0xea, 0xb7, 0x05, 0x5a, 0xc7, 0x75, 0xae, 0xbf, 0x71, 0x3d, 0x72, 0xca, 0x27, 0xc0, 0x53, - 0xd8, 0x98, 0x26, 0x7f, 0xe2, 0x3b, 0xe4, 0xba, 0x56, 0xd1, 0xbe, 0xcd, 0x16, 0x5b, 0x99, 0xa9, - 0x92, 0xed, 0x03, 0x72, 0xb9, 0x40, 0xfa, 0xb5, 0x03, 0x9f, 0x11, 0x9f, 0x09, 0x80, 0xae, 0xd9, - 0x17, 0x3b, 0xdc, 0xf9, 0x91, 0x94, 0x1b, 0xbf, 0x6a, 0x70, 0x77, 0x8a, 0x74, 0x8c, 0x19, 0xae, - 0xd5, 0x7a, 0x3a, 0xb4, 0xd3, 0xec, 0x17, 0xe5, 0x5e, 0xf2, 0xcd, 0xc7, 0xa2, 0xaa, 0x5e, 0x43, - 0xec, 0xa8, 0xaf, 0xaa, 0x01, 0xc8, 0x9d, 0xf8, 0x84, 0x38, 0x72, 0xba, 0xca, 0x63, 0x68, 0x4b, - 0xc1, 0x89, 0x63, 0x7c, 0x9e, 0xad, 0x8d, 0x0c, 0x4d, 0xe5, 0x78, 0x1f, 0xba, 0x15, 0xd9, 0x75, - 0xc6, 0x99, 0xc4, 0x3e, 0x02, 0x24, 0x8d, 0x9f, 0x07, 0xb1, 0x5f, 0x6f, 0xa6, 0xdc, 0x85, 0xb5, - 0x9c, 0x89, 0x6a, 0xec, 0x27, 0xb0, 0x2e, 0xc5, 0x2f, 0xfd, 0x49, 0x6d, 0xac, 0xcd, 0xa4, 0xac, - 0xa9, 0x91, 0x42, 0x3b, 0x4c, 0x9c, 0xe4, 0x1f, 0xb8, 0x5b, 0xc1, 0x36, 0x92, 0x08, 0xf2, 0x6f, - 0x9c, 0xf1, 0xbb, 0x06, 0x1b, 0xa6, 0x6a, 0x67, 0xf2, 0xdf, 0x0e, 0x8e, 0xec, 0xc5, 0x69, 0xcc, - 0xbc, 0x38, 0xcd, 0xe9, 0xc5, 0x19, 0x41, 0x9f, 0x06, 0x71, 0x64, 0x13, 0xcb, 0xc1, 0x0c, 0x5b, - 0x7e, 0xe0, 0x10, 0x75, 0xa0, 0x77, 0xa4, 0x9c, 0x1f, 0xe0, 0x77, 0x81, 0x43, 0x8c, 0x2d, 0xd8, - 0x2c, 0x05, 0xad, 0x12, 0xf2, 0x61, 0xf5, 0x28, 0x08, 0x6f, 0x78, 0x83, 0xd6, 0x4c, 0xa4, 0xe3, - 0x52, 0x2b, 0xb9, 0x64, 0x22, 0x93, 0xb6, 0xb9, 0xe2, 0xd2, 0x13, 0x79, 0xc3, 0xd4, 0xbe, 0x83, - 0x99, 0xdc, 0x6f, 0x24, 0xfb, 0xc7, 0x98, 0xf1, 0x7d, 0xe3, 0x29, 0xf4, 0xa7, 0xfe, 0xea, 0xf7, - 0xd6, 0x67, 0xb0, 0x6d, 0x12, 0xec, 0xc8, 0xe0, 0xc5, 0x55, 0xae, 0x3f, 0xee, 0xfe, 0xd0, 0xe0, - 0x5e, 0xb5, 0x71, 0x9d, 0x91, 0xc7, 0x2f, 0x77, 0x32, 0x52, 0x98, 0x3b, 0x21, 0x94, 0xe1, 0x49, - 0x28, 0xf2, 0x6e, 0x9a, 0x7d, 0x35, 0x57, 0xbe, 0x4f, 0xe4, 0xe5, 0x01, 0xd4, 0x28, 0x0d, 0x20, - 0x8e, 0x98, 0xd4, 0x27, 0x83, 0xd8, 0x94, 0x88, 0x8e, 0xac, 0x53, 0x0e, 0x31, 0xd5, 0x16, 0x88, - 0x2d, 0x89, 0xa8, 0x14, 0xc5, 0x48, 0xfb, 0x01, 0xe0, 0xd8, 0xa5, 0x97, 0x32, 0x2d, 0xde, 0x29, - 0x8e, 0x1b, 0xa9, 0xe7, 0x90, 0x2f, 0xb9, 0x04, 0x7b, 0x9e, 0x0a, 0x9a, 0x2f, 0xf9, 0x64, 0x88, - 0x29, 0x71, 0x54, 0x78, 0x62, 0xcd, 0x65, 0xe3, 0x88, 0x10, 0x15, 0x89, 0x58, 0x1b, 0xbf, 0x69, - 0xb0, 0xf2, 0x9c, 0x4c, 0x14, 0xf2, 0x0e, 0xc0, 0x79, 0x10, 0x05, 0x31, 0x73, 0x7d, 0x42, 0x85, - 0x83, 0x96, 0x99, 0x91, 0xfc, 0x73, 0x3f, 0x62, 0x52, 0x11, 0x6f, 0xac, 0x92, 0x13, 0x6b, 0x2e, - 0xbb, 0x20, 0x38, 0x54, 0x2f, 0x82, 0x58, 0x73, 0xa2, 0x47, 0x19, 0xb6, 0x2f, 0xc5, 0x03, 0xd0, - 0x34, 0xe5, 0xc7, 0xe1, 0xdb, 0x1e, 0x74, 0xd5, 0xdc, 0x12, 0x4c, 0x13, 0xbd, 0x86, 0x4e, 0x86, - 0xa1, 0xa2, 0xf7, 0xca, 0x44, 0xb4, 0xcc, 0x78, 0xf5, 0xf7, 0xe7, 0x68, 0xa9, 0x1b, 0xb3, 0x80, - 0x7c, 0xf8, 0x5f, 0x89, 0x01, 0xa2, 0x87, 0x65, 0xeb, 0x59, 0xfc, 0x52, 0x7f, 0x54, 0x4b, 0x37, - 0xf5, 0xc7, 0x60, 0xad, 0x82, 0xd2, 0xa1, 0xfd, 0x39, 0x28, 0x39, 0x5a, 0xa9, 0x3f, 0xae, 0xa9, - 0x9d, 0x7a, 0x7d, 0x03, 0xa8, 0xcc, 0xf7, 0xd0, 0xa3, 0xb9, 0x30, 0x53, 0x3e, 0xa9, 0xef, 0xd7, - 0x53, 0x9e, 0x99, 0xa8, 0x64, 0x82, 0x73, 0x13, 0xcd, 0x71, 0xcd, 0xb9, 0x89, 0x16, 0xe8, 0xe5, - 0x02, 0xba, 0x84, 0x7e, 0x91, 0x25, 0xa2, 0xbd, 0x59, 0x3f, 0x5d, 0x4a, 0x24, 0x54, 0x7f, 0x58, - 0x47, 0x35, 0x75, 0x66, 0x41, 0x37, 0xcb, 0xe5, 0x50, 0x45, 0xd3, 0x55, 0xb0, 0x52, 0x7d, 0x77, - 0x9e, 0x5a, 0x36, 0x9b, 0x22, 0xb7, 0xab, 0xca, 0x66, 0x06, 0x71, 0xac, 0xca, 0x66, 0x16, 0x55, - 0x34, 0x16, 0xd0, 0x4f, 0xb0, 0x5a, 0x20, 0x45, 0x68, 0x74, 0x1b, 0x40, 0x96, 0x6e, 0xe9, 0x7b, - 0x35, 0x34, 0x13, 0x4f, 0x1f, 0x6a, 0xe8, 0x1c, 0xee, 0xe4, 0xb9, 0x09, 0x7a, 0x70, 0x1b, 0x40, - 0x86, 0x58, 0xe9, 0xa3, 0xf9, 0x8a, 0x19, 0x47, 0xaf, 0xa1, 0x93, 0x21, 0x25, 0x55, 0xc3, 0xa3, - 0x4c, 0x73, 0xaa, 0x86, 0x47, 0x15, 0xb3, 0x59, 0x40, 0x67, 0xd0, 0xcb, 0xd1, 0x14, 0xb4, 0x3b, - 0xcb, 0x32, 0x4f, 0x7e, 0xf4, 0x07, 0x73, 0xf5, 0xb2, 0x4d, 0x96, 0x65, 0x2f, 0x68, 0x66, 0x70, - 0xf9, 0x01, 0xb8, 0x3b, 0x4f, 0x2d, 0x75, 0x70, 0x01, 0xab, 0x05, 0x42, 0x51, 0x75, 0xee, 0xd5, - 0x44, 0xa9, 0xea, 0xdc, 0x67, 0xb1, 0x93, 0x05, 0xf4, 0x33, 0xac, 0x57, 0xbd, 0xdd, 0xe8, 0x71, - 0x15, 0xc8, 0x4c, 0x82, 0xa0, 0x1f, 0xd4, 0x55, 0x4f, 0x1d, 0xbf, 0x84, 0x76, 0x42, 0x54, 0xd0, - 0xfd, 0xb2, 0x75, 0x81, 0x34, 0xe9, 0xc6, 0x6d, 0x2a, 0xd3, 0xe6, 0x3a, 0x5b, 0x12, 0xff, 0xc2, - 0x3c, 0xf9, 0x2b, 0x00, 0x00, 0xff, 0xff, 0x6c, 0x7d, 0x0d, 0xbb, 0x9c, 0x11, 0x00, 0x00, + // 1296 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xac, 0x58, 0x5f, 0x73, 0xdb, 0x44, + 0x10, 0x8f, 0x62, 0x3b, 0x71, 0xd6, 0x4e, 0x62, 0x2e, 0x69, 0xe2, 0x28, 0x25, 0xb8, 0x07, 0x6d, + 0x9d, 0x36, 0x0d, 0x90, 0x4e, 0xa1, 0x85, 0x17, 0x20, 0x21, 0x90, 0x87, 0xd2, 0x19, 0x85, 0x76, + 0x98, 0xa1, 0x33, 0x9a, 0x8b, 0x74, 0x49, 0x44, 0x64, 0x49, 0x95, 0x4e, 0x69, 0xc2, 0x57, 0xe0, + 0x13, 0xf0, 0xcc, 0x0b, 0xef, 0x7c, 0x20, 0x3e, 0x08, 0x2f, 0xcc, 0xfd, 0x91, 0xac, 0x7f, 0x8e, + 0xc5, 0x9f, 0xb7, 0xd3, 0xde, 0xee, 0x6f, 0x77, 0x6f, 0x6f, 0xf7, 0x7e, 0x36, 0xac, 0x5c, 0xfa, + 0x6e, 0x3c, 0xa2, 0x66, 0x44, 0xc3, 0x4b, 0x1a, 0xee, 0x06, 0xa1, 0xcf, 0x7c, 0xd4, 0xcb, 0x09, + 0xcd, 0xe0, 0x04, 0x7f, 0x08, 0xe8, 0x2b, 0xc2, 0xac, 0xf3, 0x03, 0xea, 0x52, 0x46, 0x0d, 0xfa, + 0x26, 0xa6, 0x11, 0x43, 0x1b, 0xd0, 0x3e, 0x75, 0x5c, 0x6a, 0x3a, 0x76, 0xd4, 0xd7, 0x06, 0x8d, + 0xe1, 0x82, 0x31, 0xcf, 0xbf, 0x8f, 0xec, 0x08, 0xbf, 0x80, 0x95, 0x9c, 0x41, 0x14, 0xf8, 0x5e, + 0x44, 0xd1, 0x53, 0x98, 0x0f, 0x69, 0x14, 0xbb, 0x4c, 0x1a, 0x74, 0xf6, 0xb6, 0x76, 0x8b, 0xbe, + 0x76, 0x53, 0x93, 0xd8, 0x65, 0x46, 0xa2, 0x8e, 0x1d, 0xe8, 0x66, 0x37, 0xd0, 0x3a, 0xcc, 0x2b, + 0xdf, 0x7d, 0x6d, 0xa0, 0x0d, 0x17, 0x8c, 0x39, 0xe9, 0x1a, 0xad, 0xc1, 0x5c, 0xc4, 0x08, 0x8b, + 0xa3, 0xfe, 0xec, 0x40, 0x1b, 0xb6, 0x0c, 0xf5, 0x85, 0x56, 0xa1, 0x45, 0xc3, 0xd0, 0x0f, 0xfb, + 0x0d, 0xa1, 0x2e, 0x3f, 0x10, 0x82, 0x66, 0xe4, 0xfc, 0x4c, 0xfb, 0xcd, 0x81, 0x36, 0x5c, 0x34, + 0xc4, 0x1a, 0xcf, 0x43, 0xeb, 0xeb, 0x51, 0xc0, 0xae, 0xf1, 0xa7, 0xd0, 0x7f, 0x45, 0xac, 0x38, + 0x1e, 0xbd, 0x12, 0x31, 0xee, 0x9f, 0x53, 0xeb, 0x22, 0xc9, 0x7d, 0x13, 0x16, 0x54, 0xe4, 0x2a, + 0x82, 0x45, 0xa3, 0x2d, 0x05, 0x47, 0x36, 0xfe, 0x02, 0x36, 0x2a, 0x0c, 0xd5, 0x19, 0xbc, 0x0f, + 0x8b, 0x67, 0x24, 0x3c, 0x21, 0x67, 0xd4, 0x0c, 0x09, 0x73, 0x7c, 0x61, 0xad, 0x19, 0x5d, 0x25, + 0x34, 0xb8, 0x0c, 0xff, 0x08, 0x7a, 0x0e, 0xc1, 0x1f, 0x05, 0xc4, 0x62, 0x75, 0x9c, 0xa3, 0x01, + 0x74, 0x82, 0x90, 0x12, 0xd7, 0xf5, 0x2d, 0xc2, 0xa8, 0x38, 0x85, 0x86, 0x91, 0x15, 0xe1, 0x77, + 0x61, 0xb3, 0x12, 0x5c, 0x06, 0x88, 0x9f, 0x16, 0xa2, 0xf7, 0x47, 0x23, 0xa7, 0x96, 0x6b, 0x7c, + 0xbb, 0x14, 0xb5, 0xb0, 0x54, 0xb8, 0xcf, 0x0a, 0xbb, 0x2e, 0x25, 0x5e, 0x1c, 0xd4, 0x02, 0x2e, + 0x46, 0x9c, 0x98, 0xa6, 0xc8, 0xeb, 0xf2, 0x72, 0xec, 0xfb, 0xae, 0x4b, 0x2d, 0xe6, 0xf8, 0x5e, + 0x02, 0xbb, 0x05, 0x60, 0xa5, 0x42, 0x75, 0x55, 0x32, 0x12, 0xac, 0x43, 0xbf, 0x6c, 0xaa, 0x60, + 0x7f, 0xd7, 0x60, 0xe5, 0xcb, 0x28, 0x72, 0xce, 0x3c, 0xe9, 0xb6, 0xd6, 0xf1, 0xe7, 0x1d, 0xce, + 0x16, 0x1d, 0x16, 0xcb, 0xd3, 0x28, 0x95, 0x87, 0x6b, 0x84, 0x34, 0x70, 0x1d, 0x8b, 0x08, 0x88, + 0xa6, 0x80, 0xc8, 0x8a, 0x50, 0x0f, 0x1a, 0x8c, 0xb9, 0xfd, 0x96, 0xd8, 0xe1, 0x4b, 0xbc, 0x06, + 0xab, 0xf9, 0x48, 0x55, 0x0a, 0x9f, 0xc0, 0xba, 0x94, 0x1c, 0x5f, 0x7b, 0xd6, 0xb1, 0xe8, 0x84, + 0x5a, 0x07, 0xfe, 0x97, 0x06, 0xfd, 0xb2, 0xa1, 0xba, 0xc1, 0xff, 0x35, 0xff, 0x7f, 0x9a, 0x1d, + 0x7a, 0x0f, 0x3a, 0x8c, 0x38, 0xae, 0xe9, 0x9f, 0x9e, 0x46, 0x94, 0xf5, 0xe7, 0x06, 0xda, 0xb0, + 0x69, 0x00, 0x17, 0xbd, 0x10, 0x12, 0xb4, 0x0d, 0x3d, 0x4b, 0xde, 0x62, 0x33, 0xa4, 0x97, 0x4e, + 0xc4, 0x91, 0xe7, 0x45, 0x60, 0xcb, 0x56, 0x72, 0xbb, 0xa5, 0x18, 0x61, 0x58, 0x74, 0xec, 0x2b, + 0x53, 0x0c, 0x0f, 0xd1, 0xfa, 0x6d, 0x81, 0xd6, 0x71, 0xec, 0xab, 0x43, 0xc7, 0xa5, 0xc7, 0x7c, + 0x02, 0x7c, 0x0b, 0x2b, 0x32, 0xf9, 0x43, 0xdf, 0x75, 0xfd, 0xb7, 0xb5, 0xea, 0xbe, 0x0a, 0xad, + 0xc8, 0xf1, 0x2c, 0xd9, 0x70, 0x4d, 0x43, 0x7e, 0xe0, 0x67, 0xb0, 0x9a, 0x47, 0x52, 0x47, 0x78, + 0x07, 0xba, 0x22, 0x02, 0xcb, 0xf7, 0x18, 0xf5, 0x98, 0x40, 0xeb, 0x1a, 0x1d, 0x2e, 0xdb, 0x97, + 0x22, 0xfc, 0x04, 0xd6, 0xc6, 0x15, 0x38, 0xf2, 0x6c, 0x7a, 0x55, 0xab, 0x72, 0xdf, 0x64, 0x2b, + 0xae, 0xcc, 0x94, 0xd3, 0x1d, 0x40, 0x0e, 0x17, 0x98, 0x15, 0xae, 0x7b, 0x62, 0xe7, 0x30, 0xe3, + 0xff, 0x57, 0x0d, 0x6e, 0x8d, 0x91, 0x0e, 0x08, 0x23, 0xb5, 0xce, 0x41, 0x87, 0x76, 0x5a, 0x82, + 0x59, 0xb9, 0x97, 0x7c, 0xf3, 0xd9, 0xac, 0x4a, 0xd8, 0x10, 0x3b, 0xea, 0xab, 0x6a, 0x0a, 0x73, + 0x27, 0x1e, 0xa5, 0xb6, 0x1c, 0xf1, 0xf2, 0x2e, 0xb4, 0xa5, 0xe0, 0xc8, 0xc6, 0x9f, 0x67, 0xcf, + 0x46, 0x86, 0x56, 0xff, 0x60, 0x3f, 0x06, 0x24, 0x8d, 0x9f, 0xfb, 0xb1, 0x57, 0x6f, 0xb0, 0xdd, + 0x4a, 0x2e, 0x84, 0x32, 0x51, 0xdd, 0xf5, 0x38, 0xa9, 0xee, 0x4b, 0x6f, 0x54, 0x1b, 0x6b, 0x3d, + 0x39, 0xd6, 0xd4, 0x48, 0xa1, 0xed, 0x25, 0x4e, 0xf2, 0xaf, 0xec, 0x8d, 0x60, 0x6b, 0x49, 0x04, + 0xf9, 0x87, 0x16, 0xff, 0xa1, 0xc1, 0x9a, 0xa1, 0x7a, 0x8a, 0xfe, 0xbf, 0xd3, 0x2b, 0xdb, 0xbd, + 0x8d, 0x89, 0xdd, 0xdb, 0x1c, 0x77, 0xef, 0x10, 0x7a, 0x91, 0x1f, 0x87, 0x16, 0x35, 0x6d, 0xc2, + 0x88, 0xe9, 0xf9, 0x36, 0x55, 0x05, 0x5d, 0x92, 0x72, 0x5e, 0xc0, 0xef, 0x7c, 0x9b, 0xe2, 0x0d, + 0x58, 0x2f, 0x05, 0xad, 0x12, 0xf2, 0x60, 0x79, 0xdf, 0x0f, 0xae, 0xf9, 0x05, 0xad, 0x99, 0x48, + 0xc7, 0x89, 0xcc, 0xa4, 0xd3, 0x45, 0x26, 0x6d, 0x63, 0xc1, 0x89, 0x8e, 0x64, 0x9b, 0xab, 0x7d, + 0x9b, 0x30, 0xb9, 0xdf, 0x48, 0xf6, 0x0f, 0x08, 0xe3, 0xfb, 0xf8, 0x09, 0xf4, 0xc6, 0xfe, 0xea, + 0xdf, 0xad, 0xcf, 0x60, 0xd3, 0xa0, 0xc4, 0x56, 0x3d, 0xcf, 0xe7, 0x49, 0xfd, 0x99, 0xfb, 0xa7, + 0x06, 0xb7, 0xab, 0x8d, 0xeb, 0xcc, 0x5d, 0xde, 0xdc, 0xc9, 0x5c, 0x63, 0xce, 0x88, 0x46, 0x8c, + 0x8c, 0x02, 0x35, 0x8c, 0x7a, 0x6a, 0xb8, 0x7d, 0x9f, 0xc8, 0xcb, 0x53, 0xb0, 0x51, 0x9a, 0x82, + 0x1c, 0x31, 0x39, 0x9f, 0x0c, 0x62, 0x53, 0x22, 0xda, 0xf2, 0x9c, 0x72, 0x88, 0xa9, 0xb6, 0x40, + 0x6c, 0x49, 0x44, 0xa5, 0x28, 0xe6, 0xea, 0x0f, 0x00, 0x07, 0x4e, 0x74, 0x21, 0xd3, 0xe2, 0x37, + 0xc5, 0x76, 0x42, 0xf5, 0x26, 0xf3, 0x25, 0x97, 0x10, 0xd7, 0x55, 0x41, 0xf3, 0x25, 0x9f, 0x0c, + 0x71, 0x44, 0x6d, 0x15, 0x9e, 0x58, 0x73, 0xd9, 0x69, 0x48, 0xa9, 0x8a, 0x44, 0xac, 0xf1, 0x6f, + 0x1a, 0x2c, 0x3c, 0xa7, 0x23, 0x85, 0xbc, 0x05, 0x70, 0xe6, 0x87, 0x7e, 0xcc, 0x1c, 0x8f, 0x46, + 0xc2, 0x41, 0xcb, 0xc8, 0x48, 0xfe, 0xbd, 0x1f, 0x31, 0xa9, 0xa8, 0x7b, 0xaa, 0x92, 0x13, 0x6b, + 0x2e, 0x3b, 0xa7, 0x24, 0x50, 0xcf, 0x92, 0x58, 0x8b, 0xd7, 0x80, 0x11, 0xeb, 0x42, 0xbc, 0x42, + 0xfc, 0x35, 0xe0, 0x1f, 0x7b, 0xbf, 0x2c, 0x41, 0x57, 0xcd, 0x2d, 0x41, 0x77, 0xd1, 0x6b, 0xe8, + 0x64, 0x68, 0x32, 0xfa, 0xa0, 0xcc, 0x86, 0xcb, 0xb4, 0x5b, 0xbf, 0x3b, 0x45, 0x4b, 0x75, 0xcc, + 0x0c, 0xf2, 0xe0, 0x9d, 0x12, 0x0d, 0x45, 0x0f, 0xca, 0xd6, 0x93, 0x48, 0xae, 0xfe, 0xb0, 0x96, + 0x6e, 0xea, 0x8f, 0xc1, 0x4a, 0x05, 0xaf, 0x44, 0x3b, 0x53, 0x50, 0x72, 0xdc, 0x56, 0x7f, 0x54, + 0x53, 0x3b, 0xf5, 0xfa, 0x06, 0x50, 0x99, 0x74, 0xa2, 0x87, 0x53, 0x61, 0xc6, 0xa4, 0x56, 0xdf, + 0xa9, 0xa7, 0x3c, 0x31, 0x51, 0x49, 0x47, 0xa7, 0x26, 0x9a, 0x23, 0xbc, 0x53, 0x13, 0x2d, 0x70, + 0xdc, 0x19, 0x74, 0x01, 0xbd, 0x22, 0x55, 0x45, 0xdb, 0x93, 0x7e, 0x3f, 0x95, 0x98, 0xb0, 0xfe, + 0xa0, 0x8e, 0x6a, 0xea, 0xcc, 0x84, 0x6e, 0x96, 0x50, 0xa2, 0x8a, 0x4b, 0x57, 0x41, 0x8d, 0xf5, + 0x7b, 0xd3, 0xd4, 0xb2, 0xd9, 0x14, 0x09, 0x66, 0x55, 0x36, 0x13, 0xd8, 0x6b, 0x55, 0x36, 0x93, + 0xf8, 0x2a, 0x9e, 0x41, 0x24, 0xe9, 0x3b, 0x49, 0xc3, 0xaa, 0xb2, 0xa9, 0x20, 0x7c, 0x55, 0xd9, + 0x54, 0xb1, 0x39, 0x3c, 0xf3, 0x91, 0x86, 0x7e, 0x82, 0xe5, 0x02, 0xef, 0x42, 0xc3, 0x9b, 0x62, + 0xcc, 0x32, 0x3a, 0x7d, 0xbb, 0x86, 0x66, 0xc6, 0xd7, 0x19, 0x2c, 0xe5, 0xe9, 0x0f, 0xba, 0x7f, + 0x13, 0x40, 0x86, 0xbb, 0xe9, 0xc3, 0xe9, 0x8a, 0x19, 0x47, 0xaf, 0xa1, 0x93, 0xe1, 0x3d, 0x55, + 0xf3, 0xa9, 0xcc, 0xa4, 0xf4, 0xbb, 0x53, 0xb4, 0xd2, 0xaa, 0x9c, 0xc0, 0x62, 0x8e, 0x09, 0xa1, + 0x89, 0xe7, 0x9d, 0xe7, 0x57, 0xfa, 0xfd, 0xa9, 0x7a, 0xd9, 0x7b, 0x9c, 0x25, 0x48, 0x93, 0x2b, + 0x9f, 0x9f, 0xb1, 0xf7, 0xa6, 0xa9, 0xa5, 0x0e, 0xce, 0x61, 0xb9, 0xc0, 0x59, 0xaa, 0xea, 0x5e, + 0xcd, 0xc5, 0xaa, 0xea, 0x3e, 0x89, 0x00, 0xcd, 0xa0, 0xb7, 0xb0, 0x5a, 0x45, 0x0f, 0xd0, 0xa3, + 0x2a, 0x90, 0x89, 0x1c, 0x44, 0xdf, 0xad, 0xab, 0x9e, 0x3a, 0x7e, 0x09, 0xed, 0x84, 0x0b, 0xa1, + 0x3b, 0x65, 0xeb, 0x02, 0x2f, 0xd3, 0xf1, 0x4d, 0x2a, 0xe3, 0xcb, 0x75, 0x32, 0x27, 0xfe, 0x6d, + 0x7a, 0xfc, 0x77, 0x00, 0x00, 0x00, 0xff, 0xff, 0x7d, 0xe2, 0xb6, 0x81, 0x84, 0x12, 0x00, 0x00, } diff --git a/weed/server/volume_grpc_follow.go b/weed/server/volume_grpc_follow.go new file mode 100644 index 000000000..bdd0ef6f5 --- /dev/null +++ b/weed/server/volume_grpc_follow.go @@ -0,0 +1,53 @@ +package weed_server + +import ( + "fmt" + "github.com/chrislusf/seaweedfs/weed/storage/types" + "io" + "os" + + "github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb" + "github.com/chrislusf/seaweedfs/weed/storage" +) + +func (vs *VolumeServer) VolumeFollow(req *volume_server_pb.VolumeFollowRequest, stream volume_server_pb.VolumeServer_VolumeFollowServer) error { + + v := vs.store.GetVolume(storage.VolumeId(req.VolumeId)) + if v == nil { + return fmt.Errorf("not found volume id %d", req.VolumeId) + } + + stopOffset := v.Size() + foundOffset, isLastOne, err := v.BinarySearchByAppendAtNs(req.Since) + if err != nil { + return fmt.Errorf("fail to locate by appendAtNs: %s", err) + } + + if isLastOne { + return nil + } + + startOffset := int64(foundOffset) * int64(types.NeedleEntrySize) + + buf := make([]byte, 1024*1024*2) + return sendFileContent(v.DataFile(), buf, startOffset, stopOffset, stream) + +} + +func sendFileContent(datFile *os.File, buf []byte, startOffset, stopOffset int64, stream volume_server_pb.VolumeServer_VolumeFollowServer) error { + var blockSizeLimit = int64(len(buf)) + for i := int64(0); i < stopOffset-startOffset; i += blockSizeLimit { + n, readErr := datFile.ReadAt(buf, startOffset+i) + if readErr == nil || readErr == io.EOF { + resp := &volume_server_pb.VolumeFollowResponse{} + resp.FileContent = buf[i : i+int64(n)] + sendErr := stream.Send(resp) + if sendErr != nil { + return sendErr + } + } else { + return readErr + } + } + return nil +} diff --git a/weed/storage/store_vacuum.go b/weed/storage/store_vacuum.go index cc0521491..0036315c8 100644 --- a/weed/storage/store_vacuum.go +++ b/weed/storage/store_vacuum.go @@ -20,7 +20,7 @@ func (s *Store) CompactVolume(vid VolumeId, preallocate int64) error { } func (s *Store) CommitCompactVolume(vid VolumeId) error { if v := s.findVolume(vid); v != nil { - return v.commitCompact() + return v.CommitCompact() } return fmt.Errorf("volume id %d is not found during commit compact", vid) } diff --git a/weed/storage/volume_follow.go b/weed/storage/volume_follow.go new file mode 100644 index 000000000..2aedd1682 --- /dev/null +++ b/weed/storage/volume_follow.go @@ -0,0 +1,220 @@ +package storage + +import ( + "context" + "fmt" + "github.com/chrislusf/seaweedfs/weed/operation" + "github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb" + . "github.com/chrislusf/seaweedfs/weed/storage/types" + "google.golang.org/grpc" + "io" + "os" +) + +// The volume sync with a master volume via 2 steps: +// 1. The slave checks master side to find subscription checkpoint +// to setup the replication. +// 2. The slave receives the updates from master + +/* +Assume the slave volume needs to follow the master volume. + +The master volume could be compacted, and could be many files ahead of +slave volume. + +Step 0: // implemented in command/backup.go, to avoid dat file size overflow. +0.1 If slave compact version is less than the master, do a local compaction, and set +local compact version the same as the master. +0.2 If the slave size is still bigger than the master, discard local copy and do a full copy. + +Step 1: +The slave volume ask the master by the last modification time t. +The master do a binary search in volume (use .idx as an array, and check the appendAtNs in .dat file), +to find the first entry with appendAtNs > t. + +Step 2: +The master send content bytes to the slave. The bytes are not chunked by needle. + +Step 3: +The slave generate the needle map for the new bytes. (This may be optimized to incrementally +update needle map when receiving new .dat bytes. But seems not necessary now.) + +*/ + +func (v *Volume) Follow(volumeServer string, grpcDialOption grpc.DialOption) (error) { + + ctx := context.Background() + + startFromOffset := v.Size() + appendAtNs, err := v.findLastAppendAtNs() + if err != nil { + return err + } + + err = operation.WithVolumeServerClient(volumeServer, grpcDialOption, func(client volume_server_pb.VolumeServerClient) error { + + stream, err := client.VolumeFollow(ctx, &volume_server_pb.VolumeFollowRequest{ + VolumeId: uint32(v.Id), + Since: appendAtNs, + }) + if err != nil { + return err + } + + v.dataFile.Seek(startFromOffset, io.SeekStart) + + for { + resp, recvErr := stream.Recv() + if recvErr != nil { + if recvErr == io.EOF { + break + } else { + return recvErr + } + } + + _, writeErr := v.dataFile.Write(resp.FileContent) + if writeErr != nil { + return writeErr + } + } + + return nil + + }) + + if err != nil { + return err + } + + // TODO add to needle map + + return nil +} + +func (v *Volume) findLastAppendAtNs() (uint64, error) { + offset, err := v.locateLastAppendEntry() + if err != nil { + return 0, err + } + if offset == 0 { + return 0, nil + } + return v.readAppendAtNs(offset) +} + +func (v *Volume) locateLastAppendEntry() (Offset, error) { + indexFile, e := os.OpenFile(v.FileName()+".idx", os.O_RDONLY, 0644) + if e != nil { + return 0, fmt.Errorf("cannot read %s.idx: %v", v.FileName(), e) + } + defer indexFile.Close() + + fi, err := indexFile.Stat() + if err != nil { + return 0, fmt.Errorf("file %s stat error: %v", indexFile.Name(), err) + } + fileSize := fi.Size() + if fileSize%NeedleEntrySize != 0 { + return 0, fmt.Errorf("unexpected file %s size: %d", indexFile.Name(), fileSize) + } + if fileSize == 0 { + return 0, nil + } + + bytes := make([]byte, NeedleEntrySize) + n, e := indexFile.ReadAt(bytes, fileSize-NeedleEntrySize) + if n != NeedleEntrySize { + return 0, fmt.Errorf("file %s read error: %v", indexFile.Name(), e) + } + _, offset, _ := IdxFileEntry(bytes) + + return offset, nil +} + +func (v *Volume) readAppendAtNs(offset Offset) (uint64, error) { + + n, bodyLength, err := ReadNeedleHeader(v.dataFile, v.SuperBlock.version, int64(offset)*NeedlePaddingSize) + if err != nil { + return 0, fmt.Errorf("ReadNeedleHeader: %v", err) + } + err = n.ReadNeedleBody(v.dataFile, v.SuperBlock.version, int64(offset)*NeedlePaddingSize, bodyLength) + if err != nil { + return 0, fmt.Errorf("ReadNeedleBody offset %d: %v", int64(offset)*NeedlePaddingSize, err) + } + return n.AppendAtNs, nil + +} + +// on server side +func (v *Volume) BinarySearchByAppendAtNs(sinceNs uint64) (offset Offset, isLast bool, err error) { + indexFile, openErr := os.OpenFile(v.FileName()+".idx", os.O_RDONLY, 0644) + if openErr != nil { + err = fmt.Errorf("cannot read %s.idx: %v", v.FileName(), openErr) + return + } + defer indexFile.Close() + + fi, statErr := indexFile.Stat() + if statErr != nil { + err = fmt.Errorf("file %s stat error: %v", indexFile.Name(), statErr) + return + } + fileSize := fi.Size() + if fileSize%NeedleEntrySize != 0 { + err = fmt.Errorf("unexpected file %s size: %d", indexFile.Name(), fileSize) + return + } + + bytes := make([]byte, NeedleEntrySize) + entryCount := fileSize / NeedleEntrySize + l := int64(0) + h := entryCount + + for l < h { + + m := (l + h) / 2 + + if m == entryCount { + return 0, true, nil + } + + // read the appendAtNs for entry m + offset, err = v.readAppendAtNsForIndexEntry(indexFile, bytes, m) + if err != nil { + return + } + + mNs, nsReadErr := v.readAppendAtNs(offset) + if nsReadErr != nil { + err = nsReadErr + return + } + + // move the boundary + if mNs <= sinceNs { + l = m + 1 + } else { + h = m + } + + } + + if l == entryCount { + return 0, true, nil + } + + offset, err = v.readAppendAtNsForIndexEntry(indexFile, bytes, l) + + return offset, false, err + +} + +// bytes is of size NeedleEntrySize +func (v *Volume) readAppendAtNsForIndexEntry(indexFile *os.File, bytes []byte, m int64) (Offset, error) { + if _, readErr := indexFile.ReadAt(bytes, m*NeedleEntrySize); readErr != nil && readErr != io.EOF { + return 0, readErr + } + _, offset, _ := IdxFileEntry(bytes) + return offset, nil +} diff --git a/weed/storage/volume_follow_test.go b/weed/storage/volume_follow_test.go new file mode 100644 index 000000000..3291d203a --- /dev/null +++ b/weed/storage/volume_follow_test.go @@ -0,0 +1,39 @@ +package storage + +import "testing" + +func TestBinarySearch(t *testing.T) { + var testInput []int + testInput = []int{-1, 0, 3, 5, 9, 12} + + if 3 != binarySearchForLargerThanTarget(testInput, 4) { + t.Errorf("failed to find target %d", 4) + } + if 3 != binarySearchForLargerThanTarget(testInput, 3) { + t.Errorf("failed to find target %d", 3) + } + if 6 != binarySearchForLargerThanTarget(testInput, 12) { + t.Errorf("failed to find target %d", 12) + } + if 1 != binarySearchForLargerThanTarget(testInput, -1) { + t.Errorf("failed to find target %d", -1) + } + if 0 != binarySearchForLargerThanTarget(testInput, -2) { + t.Errorf("failed to find target %d", -2) + } + +} + +func binarySearchForLargerThanTarget(nums []int, target int) int { + l := 0 + h := len(nums) + for l < h { + m := (l + h) / 2 + if nums[m] <= target { + l = m + 1 + } else { + h = m + } + } + return l +} diff --git a/weed/storage/volume_sync.go b/weed/storage/volume_sync.go index 827e6685a..7e5432417 100644 --- a/weed/storage/volume_sync.go +++ b/weed/storage/volume_sync.go @@ -26,23 +26,17 @@ Assume the slave volume needs to follow the master volume. The master volume could be compacted, and could be many files ahead of slave volume. -Step 1: -The slave volume will ask the master volume for a snapshot -of (existing file entries, last offset, number of compacted times). - -For each entry x in master existing file entries: - if x does not exist locally: - add x locally +Step 0: +If slave compact version is less than the master, do a local compaction. +If the slave size is still less than the master, discard local copy and do a full copy. -For each entry y in local slave existing file entries: - if y does not exist on master: - delete y locally +Step 1: +The slave volume ask the master by the last modification time t. +The master do a binary search in volume (use .idx as an array, and check the appendAtNs in .dat file), +to find the first entry with appendAtNs > t. Step 2: -After this, use the last offset and number of compacted times to request -the master volume to send a new file, and keep looping. If the number of -compacted times is changed, go back to step 1 (very likely this can be -optimized more later). +The master iterate following entries (including the first one) and send it to the follower. */ @@ -58,7 +52,7 @@ func (v *Volume) Synchronize(volumeServer string, grpcDialOption grpc.DialOption if err = v.Compact(0); err != nil { return fmt.Errorf("Compact Volume before synchronizing %v", err) } - if err = v.commitCompact(); err != nil { + if err = v.CommitCompact(); err != nil { return fmt.Errorf("Commit Compact before synchronizing %v", err) } } diff --git a/weed/storage/volume_vacuum.go b/weed/storage/volume_vacuum.go index b29e15843..de39628db 100644 --- a/weed/storage/volume_vacuum.go +++ b/weed/storage/volume_vacuum.go @@ -38,7 +38,7 @@ func (v *Volume) Compact2() error { return v.copyDataBasedOnIndexFile(filePath+".cpd", filePath+".cpx") } -func (v *Volume) commitCompact() error { +func (v *Volume) CommitCompact() error { glog.V(0).Infof("Committing volume %d vacuuming...", v.Id) v.dataFileAccessLock.Lock() defer v.dataFileAccessLock.Unlock() @@ -53,7 +53,7 @@ func (v *Volume) commitCompact() error { var e error if e = v.makeupDiff(v.FileName()+".cpd", v.FileName()+".cpx", v.FileName()+".dat", v.FileName()+".idx"); e != nil { - glog.V(0).Infof("makeupDiff in commitCompact volume %d failed %v", v.Id, e) + glog.V(0).Infof("makeupDiff in CommitCompact volume %d failed %v", v.Id, e) e = os.Remove(v.FileName() + ".cpd") if e != nil { return e diff --git a/weed/storage/volume_vacuum_test.go b/weed/storage/volume_vacuum_test.go index 0bc24037d..4909885b9 100644 --- a/weed/storage/volume_vacuum_test.go +++ b/weed/storage/volume_vacuum_test.go @@ -85,7 +85,7 @@ func TestCompaction(t *testing.T) { doSomeWritesDeletes(i+beforeCommitFileCount, v, t, infos) } - v.commitCompact() + v.CommitCompact() v.Close()