Mirror
/
seaweedfs
mirror of https://github.com/seaweedfs/seaweedfs.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

volume: load ec shards during heartbeats to master

pull/991/head
Chris Lu 6 years ago
parent
54b835e1ae
commit
17ac1290c0
23 changed files with 591 additions and 207 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 4
      unmaintained/see_idx/see_idx.go
  2. 2
      weed/command/server.go
  3. 13
      weed/pb/master.proto
  4. 304
      weed/pb/master_pb/master.pb.go
  5. 4
      weed/server/master_grpc_server.go
  6. 38
      weed/server/volume_grpc_client_to_master.go
  7. 69
      weed/storage/disk_location.go
  8. 84
      weed/storage/disk_location_ec.go
  9. 17
      weed/storage/disk_location_ec_test.go
  10. 0
      weed/storage/erasure_coding/3.ec07
  11. 0
      weed/storage/erasure_coding/3.ecx
  12. 4
      weed/storage/erasure_coding/ec_encoder.go
  13. 108
      weed/storage/erasure_coding/ec_volume.go
  14. 54
      weed/storage/idx/walk.go
  15. 7
      weed/storage/needle_map.go
  16. 6
      weed/storage/needle_map_leveldb.go
  17. 40
      weed/storage/needle_map_memory.go
  18. 3
      weed/storage/needle_map_metric.go
  19. 7
      weed/storage/store.go
  20. 21
      weed/storage/store_ec.go
  21. 5
      weed/storage/volume_backup.go
  22. 3
      weed/storage/volume_checking.go
  23. 5
      weed/storage/volume_vacuum.go

4
unmaintained/see_idx/see_idx.go
View File

@ -8,7 +8,7 @@ import (
"strconv"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/storage"
"github.com/chrislusf/seaweedfs/weed/storage/idx"
"github.com/chrislusf/seaweedfs/weed/storage/types"
)
@ -35,7 +35,7 @@ func main() {
}
defer indexFile.Close()
storage.WalkIndexFile(indexFile, func(key types.NeedleId, offset types.Offset, size uint32) error {
idx.WalkIndexFile(indexFile, func(key types.NeedleId, offset types.Offset, size uint32) error {
fmt.Printf("key:%v offset:%v size:%v\n", key, offset, size)
return nil
})

2
weed/command/server.go
View File

@ -222,7 +222,7 @@ func runServer(cmd *Command, args []string) bool {
glog.Fatalf("master failed to listen on grpc port %d: %v", grpcPort, err)
}
// Create your protocol servers.
glog.V(0).Infof("grpc config %+v", viper.Sub("grpc"))
glog.V(1).Infof("grpc config %+v", viper.Sub("grpc"))
grpcS := util.NewGrpcServer(security.LoadServerTLS(viper.Sub("grpc"), "master"))
master_pb.RegisterSeaweedServer(grpcS, ms)
protobuf.RegisterRaftServer(grpcS, raftServer)

13
weed/pb/master.proto
View File

@ -38,6 +38,13 @@ message Heartbeat {
// delta volumes
repeated VolumeShortInformationMessage new_volumes = 10;
repeated VolumeShortInformationMessage deleted_volumes = 11;
// erasure coding
repeated VolumeEcShardInformationMessage ec_shards = 16;
// delta erasure coding shards
repeated VolumeEcShardInformationMessage new_ec_shards = 17;
repeated VolumeEcShardInformationMessage deleted_ec_shards = 18;
}
message HeartbeatResponse {
@ -67,6 +74,12 @@ message VolumeShortInformationMessage {
uint32 ttl = 10;
}
message VolumeEcShardInformationMessage {
uint32 id = 1;
string collection = 2;
uint32 ec_index = 3;
}
message Empty {
}

304
weed/pb/master_pb/master.pb.go
View File

@ -13,6 +13,7 @@ It has these top-level messages:
HeartbeatResponse
VolumeInformationMessage
VolumeShortInformationMessage
VolumeEcShardInformationMessage
Empty
SuperBlockExtra
ClientListenRequest
@ -72,6 +73,11 @@ type Heartbeat struct {
// delta volumes
NewVolumes []*VolumeShortInformationMessage `protobuf:"bytes,10,rep,name=new_volumes,json=newVolumes" json:"new_volumes,omitempty"`
DeletedVolumes []*VolumeShortInformationMessage `protobuf:"bytes,11,rep,name=deleted_volumes,json=deletedVolumes" json:"deleted_volumes,omitempty"`
// erasure coding
EcShards []*VolumeEcShardInformationMessage `protobuf:"bytes,16,rep,name=ec_shards,json=ecShards" json:"ec_shards,omitempty"`
// delta erasure coding shards
NewEcShards []*VolumeEcShardInformationMessage `protobuf:"bytes,17,rep,name=new_ec_shards,json=newEcShards" json:"new_ec_shards,omitempty"`
DeletedEcShards []*VolumeEcShardInformationMessage `protobuf:"bytes,18,rep,name=deleted_ec_shards,json=deletedEcShards" json:"deleted_ec_shards,omitempty"`
}
func (m *Heartbeat) Reset() { *m = Heartbeat{} }
@ -156,6 +162,27 @@ func (m *Heartbeat) GetDeletedVolumes() []*VolumeShortInformationMessage {
return nil
}
func (m *Heartbeat) GetEcShards() []*VolumeEcShardInformationMessage {
if m != nil {
return m.EcShards
}
return nil
}
func (m *Heartbeat) GetNewEcShards() []*VolumeEcShardInformationMessage {
if m != nil {
return m.NewEcShards
}
return nil
}
func (m *Heartbeat) GetDeletedEcShards() []*VolumeEcShardInformationMessage {
if m != nil {
return m.DeletedEcShards
}
return nil
}
type HeartbeatResponse struct {
VolumeSizeLimit uint64 `protobuf:"varint,1,opt,name=volumeSizeLimit" json:"volumeSizeLimit,omitempty"`
Leader string `protobuf:"bytes,3,opt,name=leader" json:"leader,omitempty"`
@ -324,13 +351,45 @@ func (m *VolumeShortInformationMessage) GetTtl() uint32 {
return 0
}
type VolumeEcShardInformationMessage struct {
Id uint32 `protobuf:"varint,1,opt,name=id" json:"id,omitempty"`
Collection string `protobuf:"bytes,2,opt,name=collection" json:"collection,omitempty"`
EcIndex uint32 `protobuf:"varint,3,opt,name=ec_index,json=ecIndex" json:"ec_index,omitempty"`
}
func (m *VolumeEcShardInformationMessage) Reset() { *m = VolumeEcShardInformationMessage{} }
func (m *VolumeEcShardInformationMessage) String() string { return proto.CompactTextString(m) }
func (*VolumeEcShardInformationMessage) ProtoMessage() {}
func (*VolumeEcShardInformationMessage) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{4} }
func (m *VolumeEcShardInformationMessage) GetId() uint32 {
if m != nil {
return m.Id
}
return 0
}
func (m *VolumeEcShardInformationMessage) GetCollection() string {
if m != nil {
return m.Collection
}
return ""
}
func (m *VolumeEcShardInformationMessage) GetEcIndex() uint32 {
if m != nil {
return m.EcIndex
}
return 0
}
type Empty struct {
}
func (m *Empty) Reset() { *m = Empty{} }
func (m *Empty) String() string { return proto.CompactTextString(m) }
func (*Empty) ProtoMessage() {}
func (*Empty) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{4} }
func (*Empty) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{5} }
type SuperBlockExtra struct {
ErasureCoding *SuperBlockExtra_ErasureCoding `protobuf:"bytes,1,opt,name=erasure_coding,json=erasureCoding" json:"erasure_coding,omitempty"`
@ -339,7 +398,7 @@ type SuperBlockExtra struct {
func (m *SuperBlockExtra) Reset() { *m = SuperBlockExtra{} }
func (m *SuperBlockExtra) String() string { return proto.CompactTextString(m) }
func (*SuperBlockExtra) ProtoMessage() {}
func (*SuperBlockExtra) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{5} }
func (*SuperBlockExtra) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{6} }
func (m *SuperBlockExtra) GetErasureCoding() *SuperBlockExtra_ErasureCoding {
if m != nil {
@ -358,7 +417,7 @@ func (m *SuperBlockExtra_ErasureCoding) Reset() { *m = SuperBlockExtra_E
func (m *SuperBlockExtra_ErasureCoding) String() string { return proto.CompactTextString(m) }
func (*SuperBlockExtra_ErasureCoding) ProtoMessage() {}
func (*SuperBlockExtra_ErasureCoding) Descriptor() ([]byte, []int) {
return fileDescriptor0, []int{5, 0}
return fileDescriptor0, []int{6, 0}
}
func (m *SuperBlockExtra_ErasureCoding) GetData() uint32 {
@ -389,7 +448,7 @@ type ClientListenRequest struct {
func (m *ClientListenRequest) Reset() { *m = ClientListenRequest{} }
func (m *ClientListenRequest) String() string { return proto.CompactTextString(m) }
func (*ClientListenRequest) ProtoMessage() {}
func (*ClientListenRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{6} }
func (*ClientListenRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{7} }
func (m *ClientListenRequest) GetName() string {
if m != nil {
@ -408,7 +467,7 @@ type VolumeLocation struct {
func (m *VolumeLocation) Reset() { *m = VolumeLocation{} }
func (m *VolumeLocation) String() string { return proto.CompactTextString(m) }
func (*VolumeLocation) ProtoMessage() {}
func (*VolumeLocation) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{7} }
func (*VolumeLocation) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{8} }
func (m *VolumeLocation) GetUrl() string {
if m != nil {
@ -446,7 +505,7 @@ type LookupVolumeRequest struct {
func (m *LookupVolumeRequest) Reset() { *m = LookupVolumeRequest{} }
func (m *LookupVolumeRequest) String() string { return proto.CompactTextString(m) }
func (*LookupVolumeRequest) ProtoMessage() {}
func (*LookupVolumeRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{8} }
func (*LookupVolumeRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{9} }
func (m *LookupVolumeRequest) GetVolumeIds() []string {
if m != nil {
@ -469,7 +528,7 @@ type LookupVolumeResponse struct {
func (m *LookupVolumeResponse) Reset() { *m = LookupVolumeResponse{} }
func (m *LookupVolumeResponse) String() string { return proto.CompactTextString(m) }
func (*LookupVolumeResponse) ProtoMessage() {}
func (*LookupVolumeResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{9} }
func (*LookupVolumeResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{10} }
func (m *LookupVolumeResponse) GetVolumeIdLocations() []*LookupVolumeResponse_VolumeIdLocation {
if m != nil {
@ -488,7 +547,7 @@ func (m *LookupVolumeResponse_VolumeIdLocation) Reset() { *m = LookupVol
func (m *LookupVolumeResponse_VolumeIdLocation) String() string { return proto.CompactTextString(m) }
func (*LookupVolumeResponse_VolumeIdLocation) ProtoMessage() {}
func (*LookupVolumeResponse_VolumeIdLocation) Descriptor() ([]byte, []int) {
return fileDescriptor0, []int{9, 0}
return fileDescriptor0, []int{10, 0}
}
func (m *LookupVolumeResponse_VolumeIdLocation) GetVolumeId() string {
@ -520,7 +579,7 @@ type Location struct {
func (m *Location) Reset() { *m = Location{} }
func (m *Location) String() string { return proto.CompactTextString(m) }
func (*Location) ProtoMessage() {}
func (*Location) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{10} }
func (*Location) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{11} }
func (m *Location) GetUrl() string {
if m != nil {
@ -549,7 +608,7 @@ type AssignRequest struct {
func (m *AssignRequest) Reset() { *m = AssignRequest{} }
func (m *AssignRequest) String() string { return proto.CompactTextString(m) }
func (*AssignRequest) ProtoMessage() {}
func (*AssignRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{11} }
func (*AssignRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{12} }
func (m *AssignRequest) GetCount() uint64 {
if m != nil {
@ -612,7 +671,7 @@ type AssignResponse struct {
func (m *AssignResponse) Reset() { *m = AssignResponse{} }
func (m *AssignResponse) String() string { return proto.CompactTextString(m) }
func (*AssignResponse) ProtoMessage() {}
func (*AssignResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{12} }
func (*AssignResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{13} }
func (m *AssignResponse) GetFid() string {
if m != nil {
@ -665,7 +724,7 @@ type StatisticsRequest struct {
func (m *StatisticsRequest) Reset() { *m = StatisticsRequest{} }
func (m *StatisticsRequest) String() string { return proto.CompactTextString(m) }
func (*StatisticsRequest) ProtoMessage() {}
func (*StatisticsRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{13} }
func (*StatisticsRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{14} }
func (m *StatisticsRequest) GetReplication() string {
if m != nil {
@ -700,7 +759,7 @@ type StatisticsResponse struct {
func (m *StatisticsResponse) Reset() { *m = StatisticsResponse{} }
func (m *StatisticsResponse) String() string { return proto.CompactTextString(m) }
func (*StatisticsResponse) ProtoMessage() {}
func (*StatisticsResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{14} }
func (*StatisticsResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{15} }
func (m *StatisticsResponse) GetReplication() string {
if m != nil {
@ -752,7 +811,7 @@ type StorageType struct {
func (m *StorageType) Reset() { *m = StorageType{} }
func (m *StorageType) String() string { return proto.CompactTextString(m) }
func (*StorageType) ProtoMessage() {}
func (*StorageType) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{15} }
func (*StorageType) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{16} }
func (m *StorageType) GetReplication() string {
if m != nil {
@ -775,7 +834,7 @@ type Collection struct {
func (m *Collection) Reset() { *m = Collection{} }
func (m *Collection) String() string { return proto.CompactTextString(m) }
func (*Collection) ProtoMessage() {}
func (*Collection) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{16} }
func (*Collection) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{17} }
func (m *Collection) GetName() string {
if m != nil {
@ -790,7 +849,7 @@ type CollectionListRequest struct {
func (m *CollectionListRequest) Reset() { *m = CollectionListRequest{} }
func (m *CollectionListRequest) String() string { return proto.CompactTextString(m) }
func (*CollectionListRequest) ProtoMessage() {}
func (*CollectionListRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{17} }
func (*CollectionListRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{18} }
type CollectionListResponse struct {
Collections []*Collection `protobuf:"bytes,1,rep,name=collections" json:"collections,omitempty"`
@ -799,7 +858,7 @@ type CollectionListResponse struct {
func (m *CollectionListResponse) Reset() { *m = CollectionListResponse{} }
func (m *CollectionListResponse) String() string { return proto.CompactTextString(m) }
func (*CollectionListResponse) ProtoMessage() {}
func (*CollectionListResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{18} }
func (*CollectionListResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{19} }
func (m *CollectionListResponse) GetCollections() []*Collection {
if m != nil {
@ -815,7 +874,7 @@ type CollectionDeleteRequest struct {
func (m *CollectionDeleteRequest) Reset() { *m = CollectionDeleteRequest{} }
func (m *CollectionDeleteRequest) String() string { return proto.CompactTextString(m) }
func (*CollectionDeleteRequest) ProtoMessage() {}
func (*CollectionDeleteRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{19} }
func (*CollectionDeleteRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{20} }
func (m *CollectionDeleteRequest) GetName() string {
if m != nil {
@ -830,7 +889,7 @@ type CollectionDeleteResponse struct {
func (m *CollectionDeleteResponse) Reset() { *m = CollectionDeleteResponse{} }
func (m *CollectionDeleteResponse) String() string { return proto.CompactTextString(m) }
func (*CollectionDeleteResponse) ProtoMessage() {}
func (*CollectionDeleteResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{20} }
func (*CollectionDeleteResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{21} }
//
// volume related
@ -847,7 +906,7 @@ type DataNodeInfo struct {
func (m *DataNodeInfo) Reset() { *m = DataNodeInfo{} }
func (m *DataNodeInfo) String() string { return proto.CompactTextString(m) }
func (*DataNodeInfo) ProtoMessage() {}
func (*DataNodeInfo) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{21} }
func (*DataNodeInfo) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{22} }
func (m *DataNodeInfo) GetId() string {
if m != nil {
@ -903,7 +962,7 @@ type RackInfo struct {
func (m *RackInfo) Reset() { *m = RackInfo{} }
func (m *RackInfo) String() string { return proto.CompactTextString(m) }
func (*RackInfo) ProtoMessage() {}
func (*RackInfo) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{22} }
func (*RackInfo) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{23} }
func (m *RackInfo) GetId() string {
if m != nil {
@ -959,7 +1018,7 @@ type DataCenterInfo struct {
func (m *DataCenterInfo) Reset() { *m = DataCenterInfo{} }
func (m *DataCenterInfo) String() string { return proto.CompactTextString(m) }
func (*DataCenterInfo) ProtoMessage() {}
func (*DataCenterInfo) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{23} }
func (*DataCenterInfo) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{24} }
func (m *DataCenterInfo) GetId() string {
if m != nil {
@ -1015,7 +1074,7 @@ type TopologyInfo struct {
func (m *TopologyInfo) Reset() { *m = TopologyInfo{} }
func (m *TopologyInfo) String() string { return proto.CompactTextString(m) }
func (*TopologyInfo) ProtoMessage() {}
func (*TopologyInfo) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{24} }
func (*TopologyInfo) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{25} }
func (m *TopologyInfo) GetId() string {
if m != nil {
@ -1065,7 +1124,7 @@ type VolumeListRequest struct {
func (m *VolumeListRequest) Reset() { *m = VolumeListRequest{} }
func (m *VolumeListRequest) String() string { return proto.CompactTextString(m) }
func (*VolumeListRequest) ProtoMessage() {}
func (*VolumeListRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{25} }
func (*VolumeListRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{26} }
type VolumeListResponse struct {
TopologyInfo *TopologyInfo `protobuf:"bytes,1,opt,name=topology_info,json=topologyInfo" json:"topology_info,omitempty"`
@ -1075,7 +1134,7 @@ type VolumeListResponse struct {
func (m *VolumeListResponse) Reset() { *m = VolumeListResponse{} }
func (m *VolumeListResponse) String() string { return proto.CompactTextString(m) }
func (*VolumeListResponse) ProtoMessage() {}
func (*VolumeListResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{26} }
func (*VolumeListResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{27} }
func (m *VolumeListResponse) GetTopologyInfo() *TopologyInfo {
if m != nil {
@ -1096,6 +1155,7 @@ func init() {
proto.RegisterType((*HeartbeatResponse)(nil), "master_pb.HeartbeatResponse")
proto.RegisterType((*VolumeInformationMessage)(nil), "master_pb.VolumeInformationMessage")
proto.RegisterType((*VolumeShortInformationMessage)(nil), "master_pb.VolumeShortInformationMessage")
proto.RegisterType((*VolumeEcShardInformationMessage)(nil), "master_pb.VolumeEcShardInformationMessage")
proto.RegisterType((*Empty)(nil), "master_pb.Empty")
proto.RegisterType((*SuperBlockExtra)(nil), "master_pb.SuperBlockExtra")
proto.RegisterType((*SuperBlockExtra_ErasureCoding)(nil), "master_pb.SuperBlockExtra.ErasureCoding")
@ -1494,98 +1554,104 @@ var _Seaweed_serviceDesc = grpc.ServiceDesc{
func init() { proto.RegisterFile("master.proto", fileDescriptor0) }
var fileDescriptor0 = []byte{
// 1482 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xd4, 0x58, 0x4d, 0x6f, 0xdb, 0x46,
0x13, 0x36, 0xa9, 0x0f, 0x4b, 0xa3, 0xef, 0xb5, 0x93, 0x30, 0xca, 0x9b, 0x44, 0x61, 0x2e, 0xca,
0xfb, 0xe1, 0x37, 0x75, 0x0f, 0x3d, 0xb4, 0x45, 0x90, 0x38, 0x0e, 0x6a, 0xc4, 0x69, 0x12, 0x2a,
0x49, 0x81, 0x02, 0x05, 0xbb, 0x22, 0xd7, 0x0e, 0x61, 0x8a, 0x64, 0xc9, 0x95, 0x63, 0xe5, 0xd2,
0x43, 0x7b, 0x2c, 0xda, 0x43, 0xff, 0x44, 0x7f, 0x45, 0x2f, 0x3d, 0xe6, 0xc7, 0x14, 0xe8, 0xbd,
0x40, 0xb1, 0x1f, 0x24, 0x97, 0x94, 0x6c, 0xa7, 0x05, 0x72, 0xc8, 0x6d, 0x77, 0x66, 0x76, 0x76,
0xf6, 0x99, 0xe1, 0x33, 0x23, 0x41, 0x7b, 0x86, 0x13, 0x4a, 0xe2, 0xad, 0x28, 0x0e, 0x69, 0x88,
0x9a, 0x62, 0x67, 0x47, 0x53, 0xf3, 0x4d, 0x05, 0x9a, 0x9f, 0x11, 0x1c, 0xd3, 0x29, 0xc1, 0x14,
0x75, 0x41, 0xf7, 0x22, 0x43, 0x1b, 0x69, 0xe3, 0xa6, 0xa5, 0x7b, 0x11, 0x42, 0x50, 0x8d, 0xc2,
0x98, 0x1a, 0xfa, 0x48, 0x1b, 0x77, 0x2c, 0xbe, 0x46, 0x57, 0x01, 0xa2, 0xf9, 0xd4, 0xf7, 0x1c,
0x7b, 0x1e, 0xfb, 0x46, 0x85, 0xdb, 0x36, 0x85, 0xe4, 0x79, 0xec, 0xa3, 0x31, 0xf4, 0x67, 0xf8,
0xc4, 0x3e, 0x0e, 0xfd, 0xf9, 0x8c, 0xd8, 0x4e, 0x38, 0x0f, 0xa8, 0x51, 0xe5, 0xc7, 0xbb, 0x33,
0x7c, 0xf2, 0x82, 0x8b, 0x77, 0x98, 0x14, 0x8d, 0x58, 0x54, 0x27, 0xf6, 0x81, 0xe7, 0x13, 0xfb,
0x88, 0x2c, 0x8c, 0xda, 0x48, 0x1b, 0x57, 0x2d, 0x98, 0xe1, 0x93, 0x07, 0x9e, 0x4f, 0x1e, 0x92,
0x05, 0xba, 0x0e, 0x2d, 0x17, 0x53, 0x6c, 0x3b, 0x24, 0xa0, 0x24, 0x36, 0xea, 0xfc, 0x2e, 0x60,
0xa2, 0x1d, 0x2e, 0x61, 0xf1, 0xc5, 0xd8, 0x39, 0x32, 0xd6, 0xb9, 0x86, 0xaf, 0x59, 0x7c, 0xd8,
0x9d, 0x79, 0x81, 0xcd, 0x23, 0x6f, 0xf0, 0xab, 0x9b, 0x5c, 0xf2, 0x84, 0x85, 0xff, 0x29, 0xac,
0x8b, 0xd8, 0x12, 0xa3, 0x39, 0xaa, 0x8c, 0x5b, 0xdb, 0x37, 0xb7, 0x32, 0x34, 0xb6, 0x44, 0x78,
0x7b, 0xc1, 0x41, 0x18, 0xcf, 0x30, 0xf5, 0xc2, 0xe0, 0x11, 0x49, 0x12, 0x7c, 0x48, 0xac, 0xf4,
0x0c, 0xda, 0x83, 0x56, 0x40, 0x5e, 0xd9, 0xa9, 0x0b, 0xe0, 0x2e, 0xc6, 0x4b, 0x2e, 0x26, 0x2f,
0xc3, 0x98, 0xae, 0xf0, 0x03, 0x01, 0x79, 0xf5, 0x42, 0xba, 0x7a, 0x0a, 0x3d, 0x97, 0xf8, 0x84,
0x12, 0x37, 0x73, 0xd7, 0xfa, 0x9b, 0xee, 0xba, 0xd2, 0x81, 0x74, 0x69, 0x3e, 0x87, 0x41, 0x96,
0x4c, 0x8b, 0x24, 0x51, 0x18, 0x24, 0x04, 0x8d, 0xa1, 0x27, 0xfc, 0x4f, 0xbc, 0xd7, 0x64, 0xdf,
0x9b, 0x79, 0x94, 0x67, 0xb8, 0x6a, 0x95, 0xc5, 0xe8, 0x22, 0xd4, 0x7d, 0x82, 0x5d, 0x12, 0xcb,
0xb4, 0xca, 0x9d, 0xf9, 0xbb, 0x0e, 0xc6, 0x69, 0xd0, 0xf0, 0x9a, 0x71, 0xb9, 0xc7, 0x8e, 0xa5,
0x7b, 0x2e, 0xcb, 0x49, 0xe2, 0xbd, 0x26, 0xbc, 0x66, 0xaa, 0x16, 0x5f, 0xa3, 0x6b, 0x00, 0x4e,
0xe8, 0xfb, 0xc4, 0x61, 0x07, 0xa5, 0x73, 0x45, 0xc2, 0x72, 0xc6, 0xcb, 0x20, 0x2f, 0x97, 0xaa,
0xd5, 0x64, 0x12, 0x51, 0x29, 0x37, 0xa0, 0x2d, 0x1e, 0x2a, 0x0d, 0x44, 0xa5, 0xb4, 0x84, 0x4c,
0x98, 0xfc, 0x17, 0x50, 0x0a, 0xe6, 0x74, 0x91, 0x19, 0xd6, 0xb9, 0x61, 0x5f, 0x6a, 0xee, 0x2d,
0x52, 0xeb, 0x2b, 0xd0, 0x8c, 0x09, 0x76, 0xed, 0x30, 0xf0, 0x17, 0xbc, 0x78, 0x1a, 0x56, 0x83,
0x09, 0x1e, 0x07, 0xfe, 0x02, 0xfd, 0x07, 0x06, 0x31, 0x89, 0x7c, 0xcf, 0xc1, 0x76, 0xe4, 0x63,
0x87, 0xcc, 0x48, 0x90, 0xd6, 0x51, 0x5f, 0x2a, 0x9e, 0xa4, 0x72, 0x64, 0xc0, 0xfa, 0x31, 0x89,
0x13, 0xf6, 0xac, 0x26, 0x37, 0x49, 0xb7, 0xa8, 0x0f, 0x15, 0x4a, 0x7d, 0x03, 0xb8, 0x94, 0x2d,
0xd1, 0x2d, 0xe8, 0x3b, 0xe1, 0x2c, 0xc2, 0x0e, 0xb5, 0x63, 0x72, 0xec, 0xf1, 0x43, 0x2d, 0xae,
0xee, 0x49, 0xb9, 0x25, 0xc5, 0xe6, 0x2f, 0x1a, 0x5c, 0x3d, 0x33, 0xf5, 0x4b, 0xb0, 0x9f, 0x07,
0xf1, 0xbb, 0x7a, 0x95, 0xb9, 0x0e, 0xb5, 0xdd, 0x59, 0x44, 0x17, 0xe6, 0xaf, 0x1a, 0xf4, 0x26,
0xf3, 0x88, 0xc4, 0xf7, 0xfc, 0xd0, 0x39, 0xda, 0x3d, 0xa1, 0x31, 0x46, 0x8f, 0xa1, 0x4b, 0x62,
0x9c, 0xcc, 0x63, 0x96, 0x11, 0xd7, 0x0b, 0x0e, 0x79, 0xc4, 0xc5, 0x12, 0x2f, 0x9d, 0xd9, 0xda,
0x15, 0x07, 0x76, 0xb8, 0xbd, 0xd5, 0x21, 0xea, 0x76, 0xf8, 0x25, 0x74, 0x0a, 0x7a, 0x56, 0x6e,
0x8c, 0x10, 0x24, 0x12, 0x7c, 0xcd, 0xea, 0x38, 0xc2, 0xb1, 0x47, 0x17, 0x92, 0xb8, 0xe4, 0x8e,
0x95, 0x99, 0xe4, 0x25, 0xcf, 0x4d, 0x8c, 0xca, 0xa8, 0xc2, 0xa8, 0x41, 0x48, 0xf6, 0xdc, 0xc4,
0xbc, 0x05, 0x1b, 0x3b, 0xbe, 0x47, 0x02, 0xba, 0xef, 0x25, 0x94, 0x04, 0x16, 0xf9, 0x66, 0x4e,
0x12, 0xca, 0x6e, 0x08, 0xf0, 0x8c, 0x48, 0x5a, 0xe4, 0x6b, 0xf3, 0x5b, 0xe8, 0x8a, 0xf4, 0xec,
0x87, 0x0e, 0x4f, 0x0b, 0x03, 0x86, 0xf1, 0xa1, 0x30, 0x62, 0xcb, 0x12, 0x51, 0xea, 0x65, 0xa2,
0xbc, 0x0c, 0x0d, 0xce, 0x24, 0x79, 0x28, 0xeb, 0x8c, 0x1c, 0x3c, 0x37, 0xc9, 0xeb, 0xdd, 0x15,
0xea, 0x2a, 0x57, 0xb7, 0xd2, 0x8f, 0xdd, 0x73, 0x13, 0xf3, 0x19, 0x6c, 0xec, 0x87, 0xe1, 0xd1,
0x3c, 0x12, 0x61, 0xa4, 0xb1, 0x16, 0x5f, 0xa8, 0x8d, 0x2a, 0xec, 0xce, 0xec, 0x85, 0xa5, 0x22,
0xd1, 0xcb, 0x45, 0x62, 0xfe, 0xa1, 0xc1, 0x66, 0xd1, 0xad, 0xe4, 0x90, 0xaf, 0x61, 0x23, 0xf3,
0x6b, 0xfb, 0xf2, 0xcd, 0xe2, 0x82, 0xd6, 0xf6, 0x6d, 0x25, 0x99, 0xab, 0x4e, 0xa7, 0xb4, 0xea,
0xa6, 0x60, 0x59, 0x83, 0xe3, 0x92, 0x24, 0x19, 0x9e, 0x40, 0xbf, 0x6c, 0xc6, 0x3e, 0xd3, 0xec,
0x56, 0x89, 0x6c, 0x23, 0x3d, 0x89, 0x3e, 0x80, 0x66, 0x1e, 0x88, 0xce, 0x03, 0xd9, 0x28, 0x04,
0x22, 0xef, 0xca, 0xad, 0xd0, 0x26, 0xd4, 0x48, 0x1c, 0x87, 0x29, 0xbd, 0x89, 0x8d, 0xf9, 0x31,
0x34, 0xfe, 0x71, 0x16, 0xcd, 0x37, 0x1a, 0x74, 0xee, 0x26, 0x89, 0x77, 0x98, 0x95, 0xcb, 0x26,
0xd4, 0x04, 0xf9, 0x08, 0x92, 0x15, 0x1b, 0x34, 0x82, 0x96, 0xfc, 0xca, 0x14, 0xe8, 0x55, 0xd1,
0xb9, 0x1f, 0xb0, 0xfc, 0xf2, 0xaa, 0x22, 0x34, 0xc6, 0x27, 0xa5, 0xf6, 0x58, 0x3b, 0xb5, 0x3d,
0xd6, 0x95, 0xf6, 0x78, 0x05, 0x9a, 0xfc, 0x50, 0x10, 0xba, 0x44, 0xf6, 0xcd, 0x06, 0x13, 0x7c,
0x1e, 0xba, 0xc4, 0xfc, 0x59, 0x83, 0x6e, 0xfa, 0x1a, 0x99, 0xf9, 0x3e, 0x54, 0x0e, 0x32, 0xf4,
0xd9, 0x32, 0xc5, 0x48, 0x3f, 0x0d, 0xa3, 0xa5, 0x91, 0x20, 0x43, 0xa4, 0xaa, 0x22, 0x92, 0x25,
0xa3, 0xa6, 0x24, 0x83, 0x85, 0x8c, 0xe7, 0xf4, 0x65, 0x1a, 0x32, 0x5b, 0x9b, 0x87, 0x30, 0x98,
0x50, 0x4c, 0xbd, 0x84, 0x7a, 0x4e, 0x92, 0xc2, 0x5c, 0x02, 0x54, 0x3b, 0x0f, 0x50, 0xfd, 0x34,
0x40, 0x2b, 0x19, 0xa0, 0xe6, 0x6f, 0x1a, 0x20, 0xf5, 0x26, 0x09, 0xc1, 0x3b, 0xb8, 0x8a, 0x41,
0x46, 0x43, 0x8a, 0x7d, 0x9b, 0xf7, 0x4a, 0xd9, 0xf1, 0xb8, 0x84, 0xb5, 0x63, 0x96, 0xa5, 0x79,
0x42, 0x5c, 0xa1, 0x15, 0xed, 0xae, 0xc1, 0x04, 0x5c, 0x59, 0xec, 0x96, 0xf5, 0x52, 0xb7, 0x34,
0xef, 0x42, 0x6b, 0x42, 0xc3, 0x18, 0x1f, 0x92, 0x67, 0x8b, 0xe8, 0x6d, 0xa2, 0x97, 0xd1, 0xe9,
0x39, 0x10, 0x23, 0x80, 0x9d, 0x3c, 0xfa, 0x55, 0x04, 0x78, 0x09, 0x2e, 0xe4, 0x16, 0x8c, 0x2f,
0x65, 0x5e, 0xcc, 0xa7, 0x70, 0xb1, 0xac, 0x90, 0x30, 0x7e, 0x04, 0xad, 0x1c, 0x92, 0x94, 0x3b,
0x2e, 0x28, 0x9f, 0x6c, 0x7e, 0xce, 0x52, 0x2d, 0xcd, 0xff, 0xc1, 0xa5, 0x5c, 0x75, 0x9f, 0x93,
0xe0, 0x59, 0xdc, 0x3c, 0x04, 0x63, 0xd9, 0x5c, 0xc4, 0x60, 0xfe, 0xa4, 0x43, 0xfb, 0xbe, 0xac,
0x76, 0xd6, 0x54, 0x95, 0x36, 0xda, 0xe4, 0x6d, 0xf4, 0x06, 0xb4, 0x0b, 0xa3, 0xab, 0x98, 0x62,
0x5a, 0xc7, 0xca, 0xdc, 0xba, 0x6a, 0xc2, 0xad, 0x70, 0xb3, 0xf2, 0x84, 0xfb, 0x6f, 0x18, 0x1c,
0xc4, 0x84, 0x2c, 0x0f, 0xc3, 0x55, 0xab, 0xc7, 0x14, 0xaa, 0xed, 0x16, 0x6c, 0x60, 0x87, 0x7a,
0xc7, 0x25, 0x6b, 0x91, 0xfb, 0x81, 0x50, 0xa9, 0xf6, 0x0f, 0xb2, 0x40, 0xbd, 0xe0, 0x20, 0x4c,
0x8c, 0xfa, 0xdb, 0x0f, 0xb3, 0xf2, 0x35, 0x4c, 0x93, 0x98, 0xdf, 0xeb, 0xd0, 0xb0, 0xb0, 0x73,
0xf4, 0x7e, 0xa3, 0x71, 0x07, 0x7a, 0x19, 0xab, 0x15, 0x00, 0xb9, 0xa4, 0x00, 0xa2, 0x26, 0xde,
0xea, 0xb8, 0xca, 0x2e, 0x31, 0xff, 0xd4, 0xa0, 0x7b, 0x3f, 0x63, 0xce, 0xf7, 0x1b, 0x8c, 0x6d,
0x00, 0x46, 0xf5, 0x05, 0x1c, 0xd4, 0xd6, 0x98, 0xa6, 0xdb, 0x6a, 0xc6, 0x72, 0x95, 0x98, 0x3f,
0xea, 0xd0, 0x7e, 0x16, 0x46, 0xa1, 0x1f, 0x1e, 0x2e, 0xde, 0xef, 0xd7, 0xef, 0xc2, 0x40, 0xe9,
0x8a, 0x05, 0x10, 0x2e, 0x97, 0x8a, 0x21, 0x4f, 0xb6, 0xd5, 0x73, 0x0b, 0xfb, 0xc4, 0xdc, 0x80,
0x81, 0x9c, 0xf0, 0x14, 0x72, 0xfb, 0x4e, 0x03, 0xa4, 0x4a, 0x25, 0xb3, 0x7d, 0x02, 0x1d, 0x2a,
0xb1, 0xe3, 0xf7, 0xc9, 0x21, 0x57, 0xad, 0x3d, 0x15, 0x5b, 0xab, 0x4d, 0x55, 0xa4, 0xff, 0x0f,
0x9b, 0xf2, 0x65, 0x8c, 0xed, 0x6d, 0x9f, 0xfd, 0x14, 0xb3, 0x67, 0x53, 0x89, 0xf0, 0xa0, 0xf4,
0x23, 0xed, 0xd1, 0x74, 0xfb, 0x87, 0x1a, 0xac, 0x4f, 0x08, 0x7e, 0x45, 0x88, 0x8b, 0xf6, 0xa0,
0x33, 0x21, 0x81, 0x9b, 0xff, 0x84, 0xdf, 0x54, 0x2e, 0xcd, 0xa4, 0xc3, 0x7f, 0xad, 0x92, 0x66,
0xac, 0xb8, 0x36, 0xd6, 0x6e, 0x6b, 0xe8, 0x09, 0x74, 0x1e, 0x12, 0x12, 0xed, 0x84, 0x41, 0x40,
0x1c, 0x4a, 0x5c, 0x74, 0x4d, 0xe5, 0xe6, 0xe5, 0xc1, 0x78, 0x78, 0x79, 0x89, 0x6c, 0xd2, 0x39,
0x4a, 0x7a, 0x7c, 0x0a, 0x6d, 0x75, 0x1e, 0x2c, 0x38, 0x5c, 0x31, 0xbd, 0x0e, 0xaf, 0x9f, 0x33,
0x48, 0x9a, 0x6b, 0xe8, 0x0e, 0xd4, 0xc5, 0x80, 0x82, 0x0c, 0xc5, 0xb8, 0x30, 0x81, 0x15, 0xe2,
0x2a, 0x4e, 0x33, 0xe6, 0x1a, 0x7a, 0x08, 0x90, 0xb7, 0x78, 0xa4, 0xe2, 0xb2, 0x34, 0x63, 0x0c,
0xaf, 0x9e, 0xa2, 0xcd, 0x9c, 0x7d, 0x01, 0xdd, 0x62, 0xb3, 0x43, 0xa3, 0x95, 0xfd, 0x4c, 0xa9,
0xa1, 0xe1, 0x8d, 0x33, 0x2c, 0x32, 0xc7, 0x5f, 0x41, 0xbf, 0xdc, 0xc3, 0x90, 0xb9, 0xf2, 0x60,
0xa1, 0x1f, 0x0e, 0x6f, 0x9e, 0x69, 0xa3, 0x82, 0x90, 0x97, 0x71, 0x01, 0x84, 0xa5, 0x9a, 0x2f,
0x80, 0xb0, 0x5c, 0xfb, 0xe6, 0xda, 0xb4, 0xce, 0xff, 0x54, 0xfa, 0xf0, 0xaf, 0x00, 0x00, 0x00,
0xff, 0xff, 0x94, 0xee, 0xe2, 0x89, 0x64, 0x12, 0x00, 0x00,
// 1572 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xd4, 0x58, 0xcd, 0x6e, 0xdc, 0x46,
0x12, 0x16, 0xe7, 0x4f, 0x33, 0x35, 0x9a, 0xd1, 0x4c, 0x4b, 0xb6, 0xa9, 0xf1, 0xda, 0x1e, 0xd3,
0x97, 0xb1, 0x77, 0x57, 0xeb, 0xd5, 0x1e, 0xf6, 0xb0, 0x1b, 0x18, 0xb6, 0x2c, 0x27, 0x82, 0xe5,
0x3f, 0x8e, 0xed, 0x00, 0x01, 0x02, 0x86, 0x22, 0x4b, 0x32, 0x21, 0x0e, 0xc9, 0x90, 0x3d, 0xb2,
0xc6, 0x97, 0x1c, 0x92, 0x63, 0x90, 0x1c, 0xf2, 0x12, 0x79, 0x88, 0x20, 0x97, 0x1c, 0xf3, 0x30,
0x01, 0x72, 0x0f, 0x10, 0xf4, 0x0f, 0xc9, 0x26, 0x67, 0x24, 0xd9, 0x01, 0x7c, 0xf0, 0xad, 0xbb,
0xaa, 0xba, 0xba, 0xfa, 0xeb, 0xea, 0xaf, 0x8a, 0x84, 0x95, 0x89, 0x9d, 0x50, 0x8c, 0x37, 0xa3,
0x38, 0xa4, 0x21, 0x69, 0x89, 0x99, 0x15, 0xed, 0x1b, 0x3f, 0xd5, 0xa1, 0xf5, 0x09, 0xda, 0x31,
0xdd, 0x47, 0x9b, 0x92, 0x2e, 0x54, 0xbc, 0x48, 0xd7, 0x86, 0xda, 0xa8, 0x65, 0x56, 0xbc, 0x88,
0x10, 0xa8, 0x45, 0x61, 0x4c, 0xf5, 0xca, 0x50, 0x1b, 0x75, 0x4c, 0x3e, 0x26, 0x57, 0x00, 0xa2,
0xe9, 0xbe, 0xef, 0x39, 0xd6, 0x34, 0xf6, 0xf5, 0x2a, 0xb7, 0x6d, 0x09, 0xc9, 0x8b, 0xd8, 0x27,
0x23, 0xe8, 0x4d, 0xec, 0x13, 0xeb, 0x38, 0xf4, 0xa7, 0x13, 0xb4, 0x9c, 0x70, 0x1a, 0x50, 0xbd,
0xc6, 0x97, 0x77, 0x27, 0xf6, 0xc9, 0x4b, 0x2e, 0xde, 0x66, 0x52, 0x32, 0x64, 0x51, 0x9d, 0x58,
0x07, 0x9e, 0x8f, 0xd6, 0x11, 0xce, 0xf4, 0xfa, 0x50, 0x1b, 0xd5, 0x4c, 0x98, 0xd8, 0x27, 0x0f,
0x3c, 0x1f, 0x1f, 0xe2, 0x8c, 0x5c, 0x83, 0xb6, 0x6b, 0x53, 0xdb, 0x72, 0x30, 0xa0, 0x18, 0xeb,
0x0d, 0xbe, 0x17, 0x30, 0xd1, 0x36, 0x97, 0xb0, 0xf8, 0x62, 0xdb, 0x39, 0xd2, 0x97, 0xb9, 0x86,
0x8f, 0x59, 0x7c, 0xb6, 0x3b, 0xf1, 0x02, 0x8b, 0x47, 0xde, 0xe4, 0x5b, 0xb7, 0xb8, 0xe4, 0x29,
0x0b, 0xff, 0x23, 0x58, 0x16, 0xb1, 0x25, 0x7a, 0x6b, 0x58, 0x1d, 0xb5, 0xb7, 0x6e, 0x6c, 0x66,
0x68, 0x6c, 0x8a, 0xf0, 0x76, 0x83, 0x83, 0x30, 0x9e, 0xd8, 0xd4, 0x0b, 0x83, 0x47, 0x98, 0x24,
0xf6, 0x21, 0x9a, 0xe9, 0x1a, 0xb2, 0x0b, 0xed, 0x00, 0x5f, 0x5b, 0xa9, 0x0b, 0xe0, 0x2e, 0x46,
0x73, 0x2e, 0xc6, 0xaf, 0xc2, 0x98, 0x2e, 0xf0, 0x03, 0x01, 0xbe, 0x7e, 0x29, 0x5d, 0x3d, 0x83,
0x55, 0x17, 0x7d, 0xa4, 0xe8, 0x66, 0xee, 0xda, 0xef, 0xe8, 0xae, 0x2b, 0x1d, 0xa4, 0x2e, 0x3f,
0x86, 0x16, 0x3a, 0x56, 0xf2, 0xca, 0x8e, 0xdd, 0x44, 0xef, 0x71, 0x67, 0xb7, 0xe6, 0x9c, 0xed,
0x38, 0x63, 0x66, 0xb0, 0xc0, 0x5d, 0x13, 0x85, 0x2a, 0x21, 0x8f, 0xa1, 0xc3, 0x8e, 0x99, 0x3b,
0xeb, 0xbf, 0xb3, 0x33, 0x86, 0xd3, 0x4e, 0xea, 0xef, 0x25, 0xf4, 0xd3, 0xb3, 0xe6, 0x3e, 0xc9,
0x3b, 0xfb, 0x4c, 0x01, 0x4b, 0xfd, 0x1a, 0x2f, 0xa0, 0x9f, 0x65, 0xaf, 0x89, 0x49, 0x14, 0x06,
0x09, 0x92, 0x11, 0xac, 0x0a, 0x40, 0xc7, 0xde, 0x1b, 0xdc, 0xf3, 0x26, 0x1e, 0xe5, 0x29, 0x5d,
0x33, 0xcb, 0x62, 0x72, 0x11, 0x1a, 0x3e, 0xda, 0x2e, 0xc6, 0x32, 0x8f, 0xe5, 0xcc, 0xf8, 0xad,
0x02, 0xfa, 0x69, 0xb9, 0xc0, 0x1f, 0x89, 0xcb, 0x3d, 0x76, 0xcc, 0x8a, 0xe7, 0xb2, 0x24, 0x4c,
0xbc, 0x37, 0xc8, 0x1f, 0x49, 0xcd, 0xe4, 0x63, 0x72, 0x15, 0xc0, 0x09, 0x7d, 0x1f, 0x1d, 0xb6,
0x50, 0x3a, 0x57, 0x24, 0x2c, 0x49, 0x79, 0xde, 0xe7, 0xef, 0xa3, 0x66, 0xb6, 0x98, 0x44, 0x3c,
0x8d, 0xeb, 0xb0, 0x22, 0x4e, 0x2a, 0x0d, 0xc4, 0xd3, 0x68, 0x0b, 0x99, 0x30, 0xf9, 0x07, 0x90,
0x14, 0xd1, 0xfd, 0x59, 0x66, 0xd8, 0xe0, 0x86, 0x3d, 0xa9, 0xb9, 0x37, 0x4b, 0xad, 0x2f, 0x43,
0x2b, 0x46, 0xdb, 0xb5, 0xc2, 0xc0, 0x9f, 0xf1, 0xd7, 0xd2, 0x34, 0x9b, 0x4c, 0xf0, 0x24, 0xf0,
0x67, 0xe4, 0xef, 0xd0, 0x8f, 0x31, 0xf2, 0x3d, 0xc7, 0xb6, 0x22, 0xdf, 0x76, 0x70, 0x82, 0x41,
0xfa, 0x70, 0x7a, 0x52, 0xf1, 0x34, 0x95, 0x13, 0x1d, 0x96, 0x8f, 0x31, 0x4e, 0xd8, 0xb1, 0x5a,
0xdc, 0x24, 0x9d, 0x92, 0x1e, 0x54, 0x29, 0xf5, 0x75, 0xe0, 0x52, 0x36, 0x24, 0x37, 0xa1, 0xe7,
0x84, 0x93, 0xc8, 0x76, 0xa8, 0x15, 0xe3, 0xb1, 0xc7, 0x17, 0xb5, 0xb9, 0x7a, 0x55, 0xca, 0x4d,
0x29, 0x36, 0x7e, 0xd4, 0xe0, 0xca, 0x99, 0xb9, 0x3e, 0x07, 0xfb, 0x79, 0x10, 0xbf, 0xaf, 0x53,
0x19, 0x3e, 0x5c, 0x3b, 0x27, 0x4f, 0xcf, 0x89, 0xb5, 0x32, 0x17, 0xeb, 0x06, 0x34, 0xd1, 0xb1,
0xbc, 0xc0, 0xc5, 0x13, 0x7e, 0x92, 0x8e, 0xb9, 0x8c, 0xce, 0x2e, 0x9b, 0x1a, 0xcb, 0x50, 0xdf,
0x99, 0x44, 0x74, 0x66, 0xfc, 0xac, 0xc1, 0xea, 0x78, 0x1a, 0x61, 0x7c, 0xcf, 0x0f, 0x9d, 0xa3,
0x9d, 0x13, 0x1a, 0xdb, 0xe4, 0x09, 0x74, 0x31, 0xb6, 0x93, 0x69, 0xcc, 0xee, 0xdf, 0xf5, 0x82,
0x43, 0xbe, 0x67, 0x91, 0x41, 0x4a, 0x6b, 0x36, 0x77, 0xc4, 0x82, 0x6d, 0x6e, 0x6f, 0x76, 0x50,
0x9d, 0x0e, 0x3e, 0x83, 0x4e, 0x41, 0xcf, 0x92, 0x9b, 0xf1, 0xad, 0x3c, 0x0b, 0x1f, 0xb3, 0x57,
0x13, 0xd9, 0xb1, 0x47, 0x67, 0xb2, 0x2e, 0xc8, 0x19, 0x4b, 0x6a, 0x49, 0xfb, 0x9e, 0x9b, 0xe8,
0xd5, 0x61, 0x95, 0x31, 0xaf, 0x90, 0xec, 0xba, 0x89, 0x71, 0x13, 0xd6, 0xb6, 0x7d, 0x0f, 0x03,
0xba, 0xe7, 0x25, 0x14, 0x03, 0x13, 0xbf, 0x9c, 0x62, 0x42, 0xd9, 0x0e, 0x81, 0x3d, 0x41, 0x59,
0x75, 0xf8, 0xd8, 0xf8, 0x0a, 0xba, 0x02, 0xe2, 0xbd, 0xd0, 0xe1, 0xc0, 0xb2, 0x6b, 0x60, 0xe5,
0x46, 0x18, 0xb1, 0x61, 0xa9, 0x0e, 0x55, 0xca, 0x75, 0x68, 0x03, 0x9a, 0x9c, 0xa8, 0xf3, 0x50,
0x96, 0x19, 0xf7, 0x7a, 0x6e, 0x92, 0xbf, 0x2e, 0x57, 0xa8, 0x6b, 0x5c, 0xdd, 0x4e, 0xb9, 0xd4,
0x73, 0x13, 0xe3, 0x39, 0xac, 0xed, 0x85, 0xe1, 0xd1, 0x34, 0x12, 0x61, 0xa4, 0xb1, 0x16, 0x4f,
0xa8, 0x0d, 0xab, 0x6c, 0xcf, 0xec, 0x84, 0xe7, 0x5d, 0xb3, 0xf1, 0xbb, 0x06, 0xeb, 0x45, 0xb7,
0x92, 0xb1, 0xbe, 0x80, 0xb5, 0xcc, 0xaf, 0xe5, 0xcb, 0x33, 0x8b, 0x0d, 0xda, 0x5b, 0xb7, 0x95,
0xcb, 0x5c, 0xb4, 0x3a, 0xad, 0x5a, 0x6e, 0x0a, 0x96, 0xd9, 0x3f, 0x2e, 0x49, 0x92, 0xc1, 0x09,
0xf4, 0xca, 0x66, 0x8c, 0x14, 0xb2, 0x5d, 0x25, 0xb2, 0xcd, 0x74, 0x25, 0xf9, 0x37, 0xb4, 0xf2,
0x40, 0x2a, 0x3c, 0x90, 0xb5, 0x42, 0x20, 0x72, 0xaf, 0xdc, 0x8a, 0xac, 0x43, 0x1d, 0xe3, 0x38,
0x4c, 0xc9, 0x54, 0x4c, 0x8c, 0xff, 0x41, 0xf3, 0x2f, 0xdf, 0xa2, 0xf1, 0xab, 0x06, 0x9d, 0xbb,
0x49, 0xe2, 0x1d, 0x66, 0xe9, 0xb2, 0x0e, 0x75, 0x41, 0x75, 0x82, 0xd2, 0xc5, 0x84, 0x0c, 0xa1,
0x2d, 0xdf, 0xb4, 0x02, 0xbd, 0x2a, 0x3a, 0x97, 0x2e, 0xe4, 0x3b, 0xaf, 0x89, 0xd0, 0x18, 0x7b,
0x95, 0xba, 0x8f, 0xfa, 0xa9, 0xdd, 0x47, 0x43, 0xe9, 0x3e, 0x2e, 0x43, 0x8b, 0x2f, 0x0a, 0x42,
0x17, 0x65, 0x5b, 0xd2, 0x64, 0x82, 0xc7, 0xa1, 0x8b, 0xc6, 0x0f, 0x1a, 0x74, 0xd3, 0xd3, 0xc8,
0x9b, 0xef, 0x41, 0xf5, 0x20, 0x43, 0x9f, 0x0d, 0x53, 0x8c, 0x2a, 0xa7, 0x61, 0x34, 0xd7, 0x71,
0x65, 0x88, 0xd4, 0x54, 0x44, 0xb2, 0xcb, 0xa8, 0x2b, 0x97, 0xc1, 0x42, 0xb6, 0xa7, 0xf4, 0x55,
0x1a, 0x32, 0x1b, 0x1b, 0x87, 0xd0, 0x1f, 0x53, 0x9b, 0x7a, 0x09, 0xf5, 0x9c, 0x24, 0x85, 0xb9,
0x04, 0xa8, 0x76, 0x1e, 0xa0, 0x95, 0xd3, 0x00, 0xad, 0x66, 0x80, 0x1a, 0xbf, 0x68, 0x40, 0xd4,
0x9d, 0x24, 0x04, 0xef, 0x61, 0x2b, 0x06, 0x19, 0x0d, 0xa9, 0xed, 0x5b, 0xbc, 0x32, 0xcb, 0xfa,
0xca, 0x25, 0xac, 0xf8, 0xb3, 0x5b, 0x9a, 0x26, 0xe8, 0x0a, 0xad, 0x28, 0xae, 0x4d, 0x26, 0xe0,
0xca, 0x62, 0x6d, 0x6e, 0x94, 0x6a, 0xb3, 0x71, 0x17, 0xda, 0x63, 0x1a, 0xc6, 0xf6, 0x21, 0x3e,
0x9f, 0x45, 0x6f, 0x13, 0xbd, 0x8c, 0xae, 0x92, 0x03, 0x31, 0x04, 0xd8, 0xce, 0xa3, 0x5f, 0x44,
0x80, 0x97, 0xe0, 0x42, 0x6e, 0xc1, 0xf8, 0x52, 0xde, 0x8b, 0xf1, 0x0c, 0x2e, 0x96, 0x15, 0x12,
0xc6, 0xff, 0x42, 0x3b, 0x87, 0x24, 0xe5, 0x8e, 0x0b, 0xca, 0x93, 0xcd, 0xd7, 0x99, 0xaa, 0xa5,
0xf1, 0x4f, 0xb8, 0x94, 0xab, 0xee, 0x73, 0x12, 0x3c, 0x8b, 0x9b, 0x07, 0xa0, 0xcf, 0x9b, 0x8b,
0x18, 0x8c, 0xef, 0x2b, 0xb0, 0x72, 0x5f, 0x66, 0x3b, 0x2b, 0x8b, 0x4a, 0x21, 0x6c, 0xf1, 0x42,
0x78, 0x1d, 0x56, 0x0a, 0x5f, 0x06, 0xa2, 0x67, 0x6a, 0x1f, 0x2b, 0x9f, 0x05, 0x8b, 0x3e, 0x20,
0xaa, 0xdc, 0xac, 0xfc, 0x01, 0x71, 0x0b, 0xfa, 0x07, 0x31, 0xe2, 0xfc, 0xb7, 0x46, 0xcd, 0x5c,
0x65, 0x0a, 0xd5, 0x76, 0x13, 0xd6, 0x6c, 0x87, 0x7a, 0xc7, 0x25, 0x6b, 0x71, 0xf7, 0x7d, 0xa1,
0x52, 0xed, 0x1f, 0x64, 0x81, 0x7a, 0xc1, 0x41, 0x98, 0xe8, 0x8d, 0xb7, 0xff, 0x56, 0x90, 0xa7,
0x61, 0x9a, 0xc4, 0xf8, 0xa6, 0x02, 0x4d, 0xd3, 0x76, 0x8e, 0x3e, 0x6c, 0x34, 0xee, 0xc0, 0x6a,
0xc6, 0x6a, 0x05, 0x40, 0x2e, 0x29, 0x80, 0xa8, 0x17, 0x6f, 0x76, 0x5c, 0x65, 0x96, 0x18, 0x7f,
0x68, 0xd0, 0xbd, 0x9f, 0x31, 0xe7, 0x87, 0x0d, 0xc6, 0x16, 0x00, 0xa3, 0xfa, 0x02, 0x0e, 0x6a,
0x69, 0x4c, 0xaf, 0xdb, 0x6c, 0xc5, 0x72, 0x94, 0x18, 0xdf, 0x55, 0x60, 0xe5, 0x79, 0x18, 0x85,
0x7e, 0x78, 0x38, 0xfb, 0xb0, 0x4f, 0xbf, 0x03, 0x7d, 0xa5, 0x2a, 0x16, 0x40, 0xd8, 0x28, 0x25,
0x43, 0x7e, 0xd9, 0xe6, 0xaa, 0x5b, 0x98, 0x27, 0xc6, 0x1a, 0xf4, 0x65, 0x87, 0xa7, 0x90, 0xdb,
0xd7, 0x1a, 0x10, 0x55, 0x2a, 0x99, 0xed, 0xff, 0xd0, 0xa1, 0x12, 0x3b, 0xbe, 0x9f, 0x6c, 0x72,
0xd5, 0xdc, 0x53, 0xb1, 0x35, 0x57, 0xa8, 0x8a, 0xf4, 0xbf, 0x60, 0x5d, 0x9e, 0x8c, 0xb1, 0xbd,
0xe5, 0xb3, 0x0f, 0x3f, 0x6b, 0xb2, 0x2f, 0x11, 0xee, 0x97, 0x3e, 0x09, 0x1f, 0xed, 0x6f, 0x7d,
0x5b, 0x87, 0xe5, 0x31, 0xda, 0xaf, 0x11, 0x5d, 0xb2, 0x0b, 0x9d, 0x31, 0x06, 0x6e, 0xfe, 0x87,
0x64, 0x5d, 0xd9, 0x34, 0x93, 0x0e, 0xfe, 0xb6, 0x48, 0x9a, 0xb1, 0xe2, 0xd2, 0x48, 0xbb, 0xad,
0x91, 0xa7, 0xd0, 0x79, 0x88, 0x18, 0x6d, 0x87, 0x41, 0x80, 0x0e, 0x45, 0x97, 0x5c, 0x55, 0xb9,
0x79, 0xbe, 0x31, 0x1e, 0x6c, 0xcc, 0x91, 0x4d, 0xda, 0x47, 0x49, 0x8f, 0xcf, 0x60, 0x45, 0xed,
0x07, 0x0b, 0x0e, 0x17, 0x74, 0xaf, 0x83, 0x6b, 0xe7, 0x34, 0x92, 0xc6, 0x12, 0xb9, 0x03, 0x0d,
0xd1, 0xa0, 0x10, 0x5d, 0x31, 0x2e, 0x74, 0x60, 0x85, 0xb8, 0x8a, 0xdd, 0x8c, 0xb1, 0x44, 0x1e,
0x02, 0xe4, 0x25, 0x9e, 0xa8, 0xb8, 0xcc, 0xf5, 0x18, 0x83, 0x2b, 0xa7, 0x68, 0x33, 0x67, 0x9f,
0x42, 0xb7, 0x58, 0xec, 0xc8, 0x70, 0x61, 0x3d, 0x53, 0x72, 0x68, 0x70, 0xfd, 0x0c, 0x8b, 0xcc,
0xf1, 0xe7, 0xd0, 0x2b, 0xd7, 0x30, 0x62, 0x2c, 0x5c, 0x58, 0xa8, 0x87, 0x83, 0x1b, 0x67, 0xda,
0xa8, 0x20, 0xe4, 0x69, 0x5c, 0x00, 0x61, 0x2e, 0xe7, 0x0b, 0x20, 0xcc, 0xe7, 0xbe, 0xb1, 0xb4,
0xdf, 0xe0, 0xff, 0xec, 0xfe, 0xf3, 0x67, 0x00, 0x00, 0x00, 0xff, 0xff, 0x63, 0xc4, 0xec, 0xbe,
0xc3, 0x13, 0x00, 0x00,
}

4
weed/server/master_grpc_server.go
View File

@ -87,7 +87,7 @@ func (ms *MasterServer) SendHeartbeat(stream master_pb.Seaweed_SendHeartbeatServ
}
// update master internal volume layouts
t.IncrementalSyncDataNodeRegistration(heartbeat.NewVolumes, heartbeat.DeletedVolumes, dn)
} else {
} else if len(heartbeat.Volumes) > 0 {
// process heartbeat.Volumes
newVolumes, deletedVolumes := t.SyncDataNodeRegistration(heartbeat.Volumes, dn)
@ -99,6 +99,8 @@ func (ms *MasterServer) SendHeartbeat(stream master_pb.Seaweed_SendHeartbeatServ
glog.V(0).Infof("master see deleted volume %d from %s", uint32(v.Id), dn.Url())
message.DeletedVids = append(message.DeletedVids, uint32(v.Id))
}
} else if len(heartbeat.EcShards) > 0 {
glog.V(0).Infof("master recieved ec shards from %s: %+v", dn.Url(), heartbeat.EcShards)
}
if len(message.NewVids) > 0 || len(message.DeletedVids) > 0 {

38
weed/server/volume_grpc_client_to_master.go
View File

@ -94,7 +94,13 @@ func (vs *VolumeServer) doHeartbeat(ctx context.Context, masterNode, masterGrpcA
return "", err
}
tickChan := time.Tick(sleepInterval)
if err = stream.Send(vs.store.CollectErasureCodingHeartbeat()); err != nil {
glog.V(0).Infof("Volume Server Failed to talk with master %s: %v", masterNode, err)
return "", err
}
volumeTickChan := time.Tick(sleepInterval)
ecShardTickChan := time.Tick(17 * sleepInterval)
for {
select {
@ -109,6 +115,17 @@ func (vs *VolumeServer) doHeartbeat(ctx context.Context, masterNode, masterGrpcA
glog.V(0).Infof("Volume Server Failed to update to master %s: %v", masterNode, err)
return "", err
}
case ecShardMessage := <-vs.store.NewEcShardsChan:
deltaBeat := &master_pb.Heartbeat{
NewEcShards: []*master_pb.VolumeEcShardInformationMessage{
&ecShardMessage,
},
}
glog.V(1).Infof("volume server %s:%d adds ec shard %d:%d", vs.store.Ip, vs.store.Port, ecShardMessage.Id, ecShardMessage.EcIndex)
if err = stream.Send(deltaBeat); err != nil {
glog.V(0).Infof("Volume Server Failed to update to master %s: %v", masterNode, err)
return "", err
}
case volumeMessage := <-vs.store.DeletedVolumesChan:
deltaBeat := &master_pb.Heartbeat{
DeletedVolumes: []*master_pb.VolumeShortInformationMessage{
@ -120,12 +137,29 @@ func (vs *VolumeServer) doHeartbeat(ctx context.Context, masterNode, masterGrpcA
glog.V(0).Infof("Volume Server Failed to update to master %s: %v", masterNode, err)
return "", err
}
case <-tickChan:
case ecShardMessage := <-vs.store.DeletedEcShardsChan:
deltaBeat := &master_pb.Heartbeat{
DeletedEcShards: []*master_pb.VolumeEcShardInformationMessage{
&ecShardMessage,
},
}
glog.V(1).Infof("volume server %s:%d deletes ec shard %d:%d", vs.store.Ip, vs.store.Port, ecShardMessage.Id, ecShardMessage.EcIndex)
if err = stream.Send(deltaBeat); err != nil {
glog.V(0).Infof("Volume Server Failed to update to master %s: %v", masterNode, err)
return "", err
}
case <-volumeTickChan:
glog.V(4).Infof("volume server %s:%d heartbeat", vs.store.Ip, vs.store.Port)
if err = stream.Send(vs.store.CollectHeartbeat()); err != nil {
glog.V(0).Infof("Volume Server Failed to talk with master %s: %v", masterNode, err)
return "", err
}
case <-ecShardTickChan:
glog.V(4).Infof("volume server %s:%d ec heartbeat", vs.store.Ip, vs.store.Port)
if err = stream.Send(vs.store.CollectErasureCodingHeartbeat()); err != nil {
glog.V(0).Infof("Volume Server Failed to talk with master %s: %v", masterNode, err)
return "", err
}
case err = <-doneChan:
return
}

69
weed/storage/disk_location.go
View File

@ -9,6 +9,7 @@ import (
"fmt"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/storage/erasure_coding"
"github.com/chrislusf/seaweedfs/weed/storage/needle"
)
@ -17,43 +18,52 @@ type DiskLocation struct {
MaxVolumeCount int
volumes map[needle.VolumeId]*Volume
sync.RWMutex
// erasure coding
ecShards map[needle.VolumeId]erasure_coding.EcVolumeShards
ecShardsLock sync.RWMutex
}
func NewDiskLocation(dir string, maxVolumeCount int) *DiskLocation {
location := &DiskLocation{Directory: dir, MaxVolumeCount: maxVolumeCount}
location.volumes = make(map[needle.VolumeId]*Volume)
location.ecShards = make(map[needle.VolumeId]erasure_coding.EcVolumeShards)
return location
}
func (l *DiskLocation) volumeIdFromPath(dir os.FileInfo) (needle.VolumeId, string, error) {
name := dir.Name()
if !dir.IsDir() && strings.HasSuffix(name, ".dat") {
collection := ""
base := name[:len(name)-len(".dat")]
i := strings.LastIndex(base, "_")
if i > 0 {
collection, base = base[0:i], base[i+1:]
}
vol, err := needle.NewVolumeId(base)
return vol, collection, err
collection, volumeId, err := parseCollectionVolumeId(base)
return volumeId, collection, err
}
return 0, "", fmt.Errorf("Path is not a volume: %s", name)
}
func (l *DiskLocation) loadExistingVolume(dir os.FileInfo, needleMapKind NeedleMapType, mutex *sync.RWMutex) {
name := dir.Name()
if !dir.IsDir() && strings.HasSuffix(name, ".dat") {
vid, collection, err := l.volumeIdFromPath(dir)
func parseCollectionVolumeId(base string) (collection string, vid needle.VolumeId, err error) {
i := strings.LastIndex(base, "_")
if i > 0 {
collection, base = base[0:i], base[i+1:]
}
vol, err := needle.NewVolumeId(base)
return collection, vol, err
}
func (l *DiskLocation) loadExistingVolume(fileInfo os.FileInfo, needleMapKind NeedleMapType) {
name := fileInfo.Name()
if !fileInfo.IsDir() && strings.HasSuffix(name, ".dat") {
vid, collection, err := l.volumeIdFromPath(fileInfo)
if err == nil {
mutex.RLock()
l.RLock()
_, found := l.volumes[vid]
mutex.RUnlock()
l.RUnlock()
if !found {
if v, e := NewVolume(l.Directory, collection, vid, needleMapKind, nil, nil, 0); e == nil {
mutex.Lock()
l.Lock()
l.volumes[vid] = v
mutex.Unlock()
l.Unlock()
size, _, _ := v.FileStat()
glog.V(0).Infof("data file %s, replicaPlacement=%s v=%d size=%d ttl=%s",
l.Directory+"/"+name, v.ReplicaPlacement, v.Version(), size, v.Ttl.String())
@ -80,13 +90,12 @@ func (l *DiskLocation) concurrentLoadingVolumes(needleMapKind NeedleMapType, con
}()
var wg sync.WaitGroup
var mutex sync.RWMutex
for workerNum := 0; workerNum < concurrency; workerNum++ {
wg.Add(1)
go func() {
defer wg.Done()
for dir := range task_queue {
l.loadExistingVolume(dir, needleMapKind, &mutex)
l.loadExistingVolume(dir, needleMapKind)
}
}()
}
@ -95,12 +104,13 @@ func (l *DiskLocation) concurrentLoadingVolumes(needleMapKind NeedleMapType, con
}
func (l *DiskLocation) loadExistingVolumes(needleMapKind NeedleMapType) {
l.Lock()
defer l.Unlock()
l.concurrentLoadingVolumes(needleMapKind, 10)
glog.V(0).Infof("Store started on dir: %s with %d volumes max %d", l.Directory, len(l.volumes), l.MaxVolumeCount)
l.loadAllEcShards()
glog.V(0).Infof("Store started on dir: %s with %d ec shards", l.Directory, len(l.ecShards))
glog.V(0).Infoln("Store started on dir:", l.Directory, "with", len(l.volumes), "volumes", "max", l.MaxVolumeCount)
}
func (l *DiskLocation) DeleteCollectionFromDiskLocation(collection string) (e error) {
@ -132,12 +142,11 @@ func (l *DiskLocation) deleteVolumeById(vid needle.VolumeId) (e error) {
}
func (l *DiskLocation) LoadVolume(vid needle.VolumeId, needleMapKind NeedleMapType) bool {
if dirs, err := ioutil.ReadDir(l.Directory); err == nil {
for _, dir := range dirs {
volId, _, err := l.volumeIdFromPath(dir)
if fileInfos, err := ioutil.ReadDir(l.Directory); err == nil {
for _, fileInfo := range fileInfos {
volId, _, err := l.volumeIdFromPath(fileInfo)
if vid == volId && err == nil {
var mutex sync.RWMutex
l.loadExistingVolume(dir, needleMapKind, &mutex)
l.loadExistingVolume(fileInfo, needleMapKind)
return true
}
}
@ -194,10 +203,16 @@ func (l *DiskLocation) VolumesLen() int {
func (l *DiskLocation) Close() {
l.Lock()
defer l.Unlock()
for _, v := range l.volumes {
v.Close()
}
l.Unlock()
l.ecShardsLock.Lock()
for _, shards := range l.ecShards {
shards.Close()
}
l.ecShardsLock.Unlock()
return
}

84
weed/storage/disk_location_ec.go
View File

@ -0,0 +1,84 @@
package storage
import (
"fmt"
"io/ioutil"
"path"
"regexp"
"sort"
"strconv"
"github.com/chrislusf/seaweedfs/weed/storage/erasure_coding"
"github.com/chrislusf/seaweedfs/weed/storage/needle"
)
var (
re = regexp.MustCompile("\\.ec[0-9][0-9]")
)
func (l *DiskLocation) loadEcShards(baseName string, shards []string, collection string, vid needle.VolumeId) (err error){
for _, shard := range shards{
shardId, err := strconv.ParseInt(path.Ext(shard)[3:], 10, 64)
if err != nil {
return fmt.Errorf("failed to parse ec shard name %v: %v", shard, err)
}
ecVolumeShard, err := erasure_coding.NewEcVolumeShard(l.Directory, collection, vid, int(shardId))
if err != nil {
return fmt.Errorf("failed to create ec shard %v: %v", shard, err)
}
l.ecShardsLock.Lock()
l.ecShards[vid] = append(l.ecShards[vid], ecVolumeShard)
l.ecShardsLock.Unlock()
}
return nil
}
func (l *DiskLocation) loadAllEcShards() (err error){
fileInfos, err := ioutil.ReadDir(l.Directory)
if err != nil {
return fmt.Errorf("load all ec shards in dir %s: %v", l.Directory, err)
}
sort.Slice(fileInfos, func(i, j int) bool {
return fileInfos[i].Name() < fileInfos[j].Name()
})
var sameVolumeShards []string
var prevVolumeId needle.VolumeId
for _, fileInfo := range fileInfos{
if fileInfo.IsDir(){
continue
}
ext := path.Ext(fileInfo.Name())
name := fileInfo.Name()
baseName := name[:len(name)-len(ext)]
collection, volumeId, err := parseCollectionVolumeId(baseName)
if err != nil {
continue
}
if re.MatchString(ext){
if prevVolumeId == 0 || volumeId == prevVolumeId {
sameVolumeShards = append(sameVolumeShards, fileInfo.Name())
}else{
sameVolumeShards = []string{fileInfo.Name()}
}
prevVolumeId = volumeId
continue
}
if ext == ".ecx" && volumeId == prevVolumeId{
if err = l.loadEcShards(baseName, sameVolumeShards, collection, volumeId);err!=nil{
return fmt.Errorf("loadEcShards collection:%v volumeId:%d : %v", collection, volumeId, err)
}
prevVolumeId = volumeId
continue
}
}
return nil
}

17
weed/storage/disk_location_ec_test.go
View File

@ -0,0 +1,17 @@
package storage
import (
"testing"
)
func TestLoadingEcShards(t *testing.T) {
dl := NewDiskLocation("./erasure_coding", 100)
err := dl.loadAllEcShards()
if err != nil {
t.Errorf("load all ec shards: %v", err)
}
if len(dl.ecShards)!=1 {
t.Errorf("loading err")
}
}

0
weed/storage/erasure_coding/3.ec07
View File

0
weed/storage/erasure_coding/3.ecx
View File

4
weed/storage/erasure_coding/ec_encoder.go
View File

@ -6,7 +6,7 @@ import (
"os"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/storage"
"github.com/chrislusf/seaweedfs/weed/storage/idx"
"github.com/chrislusf/seaweedfs/weed/storage/needle_map"
"github.com/chrislusf/seaweedfs/weed/storage/types"
"github.com/klauspost/reedsolomon"
@ -190,7 +190,7 @@ func readCompactMap(baseFileName string) (*needle_map.CompactMap, error) {
defer indexFile.Close()
cm := needle_map.NewCompactMap()
err = storage.WalkIndexFile(indexFile, func(key types.NeedleId, offset types.Offset, size uint32) error {
err = idx.WalkIndexFile(indexFile, func(key types.NeedleId, offset types.Offset, size uint32) error {
if !offset.IsZero() && size != types.TombstoneFileSize {
cm.Set(key, offset, size)
} else {

108
weed/storage/erasure_coding/ec_volume.go
View File

@ -0,0 +1,108 @@
package erasure_coding
import (
"fmt"
"os"
"path"
"strconv"
"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
"github.com/chrislusf/seaweedfs/weed/storage/needle"
)
type EcVolumeShard struct {
VolumeId needle.VolumeId
ShardId uint8
Collection string
dir string
ecdFile *os.File
ecxFile *os.File
}
type EcVolumeShards []*EcVolumeShard
func NewEcVolumeShard(dirname string, collection string, id needle.VolumeId, shardId int) (v *EcVolumeShard, e error) {
v = &EcVolumeShard{dir: dirname, Collection: collection, VolumeId: id, ShardId: uint8(shardId)}
baseFileName := v.FileName()
if v.ecxFile, e = os.OpenFile(baseFileName+".ecx", os.O_RDONLY, 0644); e != nil {
return nil, fmt.Errorf("cannot read ec volume index %s.ecx: %v", baseFileName, e)
}
if v.ecdFile, e = os.OpenFile(baseFileName+ToExt(shardId), os.O_RDONLY, 0644); e != nil {
return nil, fmt.Errorf("cannot read ec volume shard %s.%s: %v", baseFileName, ToExt(shardId), e)
}
return
}
func (shards *EcVolumeShards) AddEcVolumeShard(ecVolumeShard *EcVolumeShard) bool {
for _, s := range *shards {
if s.ShardId == ecVolumeShard.ShardId {
return false
}
}
*shards = append(*shards, ecVolumeShard)
return true
}
func (shards *EcVolumeShards) DeleteEcVolumeShard(ecVolumeShard *EcVolumeShard) bool {
foundPosition := -1
for i, s := range *shards {
if s.ShardId == ecVolumeShard.ShardId {
foundPosition = i
}
}
if foundPosition < 0 {
return false
}
*shards = append((*shards)[:foundPosition], (*shards)[foundPosition+1:]...)
return true
}
func (shards *EcVolumeShards) Close() {
for _, s := range *shards {
s.Close()
}
}
func (shards *EcVolumeShards) ToVolumeInformationMessage() (messages []*master_pb.VolumeEcShardInformationMessage) {
for _, s := range *shards {
m := &master_pb.VolumeEcShardInformationMessage{
Id: uint32(s.VolumeId),
Collection: s.Collection,
EcIndex: uint32(s.ShardId),
}
messages = append(messages, m)
}
return
}
func (v *EcVolumeShard) String() string {
return fmt.Sprintf("ec shard %v:%v, dir:%s, Collection:%s", v.VolumeId, v.ShardId, v.dir, v.Collection)
}
func (v *EcVolumeShard) FileName() (fileName string) {
return EcShardFileName(v.Collection, v.dir, int(v.VolumeId))
}
func EcShardFileName(collection string, dir string, id int) (fileName string) {
idString := strconv.Itoa(id)
if collection == "" {
fileName = path.Join(dir, idString)
} else {
fileName = path.Join(dir, collection+"_"+idString)
}
return
}
func (v *EcVolumeShard) Close() {
if v.ecdFile != nil {
_ = v.ecdFile.Close()
v.ecdFile = nil
}
if v.ecxFile != nil {
_ = v.ecxFile.Close()
v.ecxFile = nil
}
}

54
weed/storage/idx/walk.go
View File

@ -0,0 +1,54 @@
package idx
import (
"io"
"os"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/storage/types"
"github.com/chrislusf/seaweedfs/weed/util"
)
// walks through the index file, calls fn function with each key, offset, size
// stops with the error returned by the fn function
func WalkIndexFile(r *os.File, fn func(key types.NeedleId, offset types.Offset, size uint32) error) error {
var readerOffset int64
bytes := make([]byte, types.NeedleMapEntrySize*RowsToRead)
count, e := r.ReadAt(bytes, readerOffset)
glog.V(3).Infoln("file", r.Name(), "readerOffset", readerOffset, "count", count, "e", e)
readerOffset += int64(count)
var (
key types.NeedleId
offset types.Offset
size uint32
i int
)
for count > 0 && e == nil || e == io.EOF {
for i = 0; i+types.NeedleMapEntrySize <= count; i += types.NeedleMapEntrySize {
key, offset, size = IdxFileEntry(bytes[i : i+types.NeedleMapEntrySize])
if e = fn(key, offset, size); e != nil {
return e
}
}
if e == io.EOF {
return nil
}
count, e = r.ReadAt(bytes, readerOffset)
glog.V(3).Infoln("file", r.Name(), "readerOffset", readerOffset, "count", count, "e", e)
readerOffset += int64(count)
}
return e
}
func IdxFileEntry(bytes []byte) (key types.NeedleId, offset types.Offset, size uint32) {
key = types.BytesToNeedleId(bytes[:types.NeedleIdSize])
offset = types.BytesToOffset(bytes[types.NeedleIdSize : types.NeedleIdSize+types.OffsetSize])
size = util.BytesToUint32(bytes[types.NeedleIdSize+types.OffsetSize : types.NeedleIdSize+types.OffsetSize+types.SizeSize])
return
}
const (
RowsToRead = 1024
)

7
weed/storage/needle_map.go
View File

@ -8,7 +8,6 @@ import (
"github.com/chrislusf/seaweedfs/weed/storage/needle_map"
. "github.com/chrislusf/seaweedfs/weed/storage/types"
"github.com/chrislusf/seaweedfs/weed/util"
)
type NeedleMapType int
@ -55,12 +54,6 @@ func (nm *baseNeedleMapper) IndexFileName() string {
return nm.indexFile.Name()
}
func IdxFileEntry(bytes []byte) (key NeedleId, offset Offset, size uint32) {
key = BytesToNeedleId(bytes[:NeedleIdSize])
offset = BytesToOffset(bytes[NeedleIdSize : NeedleIdSize+OffsetSize])
size = util.BytesToUint32(bytes[NeedleIdSize+OffsetSize : NeedleIdSize+OffsetSize+SizeSize])
return
}
func (nm *baseNeedleMapper) appendToIndexFile(key NeedleId, offset Offset, size uint32) error {
bytes := needle_map.ToBytes(key, offset, size)

6
weed/storage/needle_map_leveldb.go
View File

@ -2,10 +2,12 @@ package storage
import (
"fmt"
"github.com/syndtr/goleveldb/leveldb/opt"
"os"
"path/filepath"
"github.com/chrislusf/seaweedfs/weed/storage/idx"
"github.com/syndtr/goleveldb/leveldb/opt"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/storage/needle_map"
. "github.com/chrislusf/seaweedfs/weed/storage/types"
@ -64,7 +66,7 @@ func generateLevelDbFile(dbFileName string, indexFile *os.File) error {
return err
}
defer db.Close()
return WalkIndexFile(indexFile, func(key NeedleId, offset Offset, size uint32) error {
return idx.WalkIndexFile(indexFile, func(key NeedleId, offset Offset, size uint32) error {
if !offset.IsZero() && size != TombstoneFileSize {
levelDbWrite(db, key, offset, size)
} else {

40
weed/storage/needle_map_memory.go
View File

@ -1,10 +1,10 @@
package storage
import (
"io"
"os"
"github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/storage/idx"
"github.com/chrislusf/seaweedfs/weed/storage/needle_map"
. "github.com/chrislusf/seaweedfs/weed/storage/types"
)
@ -30,10 +30,6 @@ func NewBtreeNeedleMap(file *os.File) *NeedleMap {
return nm
}
const (
RowsToRead = 1024
)
func LoadCompactNeedleMap(file *os.File) (*NeedleMap, error) {
nm := NewCompactNeedleMap(file)
return doLoading(file, nm)
@ -45,7 +41,7 @@ func LoadBtreeNeedleMap(file *os.File) (*NeedleMap, error) {
}
func doLoading(file *os.File, nm *NeedleMap) (*NeedleMap, error) {
e := WalkIndexFile(file, func(key NeedleId, offset Offset, size uint32) error {
e := idx.WalkIndexFile(file, func(key NeedleId, offset Offset, size uint32) error {
nm.MaybeSetMaxFileKey(key)
if !offset.IsZero() && size != TombstoneFileSize {
nm.FileCounter++
@ -68,38 +64,6 @@ func doLoading(file *os.File, nm *NeedleMap) (*NeedleMap, error) {
return nm, e
}
// walks through the index file, calls fn function with each key, offset, size
// stops with the error returned by the fn function
func WalkIndexFile(r *os.File, fn func(key NeedleId, offset Offset, size uint32) error) error {
var readerOffset int64
bytes := make([]byte, NeedleMapEntrySize*RowsToRead)
count, e := r.ReadAt(bytes, readerOffset)
glog.V(3).Infoln("file", r.Name(), "readerOffset", readerOffset, "count", count, "e", e)
readerOffset += int64(count)
var (
key NeedleId
offset Offset
size uint32
i int
)
for count > 0 && e == nil || e == io.EOF {
for i = 0; i+NeedleMapEntrySize <= count; i += NeedleMapEntrySize {
key, offset, size = IdxFileEntry(bytes[i : i+NeedleMapEntrySize])
if e = fn(key, offset, size); e != nil {
return e
}
}
if e == io.EOF {
return nil
}
count, e = r.ReadAt(bytes, readerOffset)
glog.V(3).Infoln("file", r.Name(), "readerOffset", readerOffset, "count", count, "e", e)
readerOffset += int64(count)
}
return e
}
func (nm *NeedleMap) Put(key NeedleId, offset Offset, size uint32) error {
_, oldSize := nm.m.Set(NeedleId(key), offset, size)
nm.logPut(key, oldSize, size)

3
weed/storage/needle_map_metric.go
View File

@ -5,6 +5,7 @@ import (
"os"
"sync/atomic"
"github.com/chrislusf/seaweedfs/weed/storage/idx"
. "github.com/chrislusf/seaweedfs/weed/storage/types"
"github.com/willf/bloom"
)
@ -119,7 +120,7 @@ func reverseWalkIndexFile(r *os.File, initFn func(entryCount int64), fn func(key
return e
}
for i := int(nextBatchSize) - 1; i >= 0; i-- {
key, offset, size := IdxFileEntry(bytes[i*NeedleMapEntrySize : i*NeedleMapEntrySize+NeedleMapEntrySize])
key, offset, size := idx.IdxFileEntry(bytes[i*NeedleMapEntrySize : i*NeedleMapEntrySize+NeedleMapEntrySize])
if e = fn(key, offset, size); e != nil {
return e
}

7
weed/storage/store.go
View File

@ -30,6 +30,8 @@ type Store struct {
NeedleMapType NeedleMapType
NewVolumesChan chan master_pb.VolumeShortInformationMessage
DeletedVolumesChan chan master_pb.VolumeShortInformationMessage
NewEcShardsChan chan master_pb.VolumeEcShardInformationMessage
DeletedEcShardsChan chan master_pb.VolumeEcShardInformationMessage
}
func (s *Store) String() (str string) {
@ -47,6 +49,10 @@ func NewStore(port int, ip, publicUrl string, dirnames []string, maxVolumeCounts
}
s.NewVolumesChan = make(chan master_pb.VolumeShortInformationMessage, 3)
s.DeletedVolumesChan = make(chan master_pb.VolumeShortInformationMessage, 3)
s.NewEcShardsChan = make(chan master_pb.VolumeEcShardInformationMessage, 3)
s.DeletedEcShardsChan = make(chan master_pb.VolumeEcShardInformationMessage, 3)
return
}
func (s *Store) AddVolume(volumeId needle.VolumeId, collection string, needleMapKind NeedleMapType, replicaPlacement string, ttlString string, preallocate int64) error {
@ -186,6 +192,7 @@ func (s *Store) CollectHeartbeat() *master_pb.Heartbeat {
}
}
func (s *Store) Close() {
for _, location := range s.Locations {
location.Close()

21
weed/storage/store_ec.go
View File

@ -0,0 +1,21 @@
package storage
import (
"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
)
func (s *Store) CollectErasureCodingHeartbeat() *master_pb.Heartbeat {
var ecShardMessages []*master_pb.VolumeEcShardInformationMessage
for _, location := range s.Locations {
location.ecShardsLock.RLock()
for _, ecShards := range location.ecShards {
ecShardMessages = append(ecShardMessages, ecShards.ToVolumeInformationMessage()...)
}
location.ecShardsLock.RUnlock()
}
return &master_pb.Heartbeat{
EcShards: ecShardMessages,
}
}

5
weed/storage/volume_backup.go
View File

@ -8,6 +8,7 @@ import (
"github.com/chrislusf/seaweedfs/weed/operation"
"github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb"
"github.com/chrislusf/seaweedfs/weed/storage/idx"
"github.com/chrislusf/seaweedfs/weed/storage/needle"
. "github.com/chrislusf/seaweedfs/weed/storage/types"
"google.golang.org/grpc"
@ -142,7 +143,7 @@ func (v *Volume) locateLastAppendEntry() (Offset, error) {
if n != NeedleMapEntrySize {
return Offset{}, fmt.Errorf("file %s read error: %v", indexFile.Name(), e)
}
_, offset, _ := IdxFileEntry(bytes)
_, offset, _ := idx.IdxFileEntry(bytes)
return offset, nil
}
@ -230,7 +231,7 @@ func (v *Volume) readAppendAtNsForIndexEntry(indexFile *os.File, bytes []byte, m
if _, readErr := indexFile.ReadAt(bytes, m*NeedleMapEntrySize); readErr != nil && readErr != io.EOF {
return Offset{}, readErr
}
_, offset, _ := IdxFileEntry(bytes)
_, offset, _ := idx.IdxFileEntry(bytes)
return offset, nil
}

3
weed/storage/volume_checking.go
View File

@ -4,6 +4,7 @@ import (
"fmt"
"os"
"github.com/chrislusf/seaweedfs/weed/storage/idx"
"github.com/chrislusf/seaweedfs/weed/storage/needle"
. "github.com/chrislusf/seaweedfs/weed/storage/types"
"github.com/chrislusf/seaweedfs/weed/util"
@ -21,7 +22,7 @@ func CheckVolumeDataIntegrity(v *Volume, indexFile *os.File) (lastAppendAtNs uin
if lastIdxEntry, e = readIndexEntryAtOffset(indexFile, indexSize-NeedleMapEntrySize); e != nil {
return 0, fmt.Errorf("readLastIndexEntry %s failed: %v", indexFile.Name(), e)
}
key, offset, size := IdxFileEntry(lastIdxEntry)
key, offset, size := idx.IdxFileEntry(lastIdxEntry)
if offset.IsZero() {
return 0, nil
}

5
weed/storage/volume_vacuum.go
View File

@ -6,6 +6,7 @@ import (
"time"
"github.com/chrislusf/seaweedfs/weed/glog"
idx2 "github.com/chrislusf/seaweedfs/weed/storage/idx"
"github.com/chrislusf/seaweedfs/weed/storage/needle"
. "github.com/chrislusf/seaweedfs/weed/storage/types"
"github.com/chrislusf/seaweedfs/weed/util"
@ -143,7 +144,7 @@ func (v *Volume) makeupDiff(newDatFileName, newIdxFileName, oldDatFileName, oldI
if IdxEntry, err = readIndexEntryAtOffset(oldIdxFile, idxOffset); err != nil {
return fmt.Errorf("readIndexEntry %s at offset %d failed: %v", oldIdxFileName, idxOffset, err)
}
key, offset, size := IdxFileEntry(IdxEntry)
key, offset, size := idx2.IdxFileEntry(IdxEntry)
glog.V(4).Infof("key %d offset %d size %d", key, offset, size)
if _, found := incrementedHasUpdatedIndexEntry[key]; !found {
incrementedHasUpdatedIndexEntry[key] = keyField{
@ -329,7 +330,7 @@ func (v *Volume) copyDataBasedOnIndexFile(dstName, idxName string) (err error) {
dst.Write(v.SuperBlock.Bytes())
newOffset := int64(v.SuperBlock.BlockSize())
WalkIndexFile(oldIndexFile, func(key NeedleId, offset Offset, size uint32) error {
idx2.WalkIndexFile(oldIndexFile, func(key NeedleId, offset Offset, size uint32) error {
if offset.IsZero() || size == TombstoneFileSize {
return nil
}

Write Preview
Loading…
Cancel
Save