Chris Lu
8 years ago
20 changed files with 664 additions and 483 deletions
-
27weed/command/master.go
-
28weed/command/server.go
-
2weed/command/volume.go
-
203weed/operation/system_message.pb.go
-
59weed/operation/system_message_test.go
-
6weed/pb/Makefile
-
384weed/pb/seaweed.pb.go
-
41weed/pb/seaweed.proto
-
57weed/server/master_grpc_server.go
-
1weed/server/master_server.go
-
36weed/server/master_server_handlers_admin.go
-
74weed/server/volume_grpc_client.go
-
34weed/server/volume_server.go
-
100weed/storage/store.go
-
25weed/storage/volume_info.go
-
3weed/topology/data_node.go
-
6weed/topology/node.go
-
5weed/topology/rack.go
-
38weed/topology/topology.go
-
14weed/topology/topology_event_handling.go
@ -1,203 +0,0 @@ |
|||
// Code generated by protoc-gen-go.
|
|||
// source: system_message.proto
|
|||
// DO NOT EDIT!
|
|||
|
|||
/* |
|||
Package operation is a generated protocol buffer package. |
|||
|
|||
It is generated from these files: |
|||
system_message.proto |
|||
|
|||
It has these top-level messages: |
|||
VolumeInformationMessage |
|||
JoinMessage |
|||
*/ |
|||
package operation |
|||
|
|||
import proto "github.com/golang/protobuf/proto" |
|||
import math "math" |
|||
|
|||
// Reference imports to suppress errors if they are not otherwise used.
|
|||
var _ = proto.Marshal |
|||
var _ = math.Inf |
|||
|
|||
type VolumeInformationMessage struct { |
|||
Id *uint32 `protobuf:"varint,1,req,name=id" json:"id,omitempty"` |
|||
Size *uint64 `protobuf:"varint,2,req,name=size" json:"size,omitempty"` |
|||
Collection *string `protobuf:"bytes,3,opt,name=collection" json:"collection,omitempty"` |
|||
FileCount *uint64 `protobuf:"varint,4,req,name=file_count" json:"file_count,omitempty"` |
|||
DeleteCount *uint64 `protobuf:"varint,5,req,name=delete_count" json:"delete_count,omitempty"` |
|||
DeletedByteCount *uint64 `protobuf:"varint,6,req,name=deleted_byte_count" json:"deleted_byte_count,omitempty"` |
|||
ReadOnly *bool `protobuf:"varint,7,opt,name=read_only" json:"read_only,omitempty"` |
|||
ReplicaPlacement *uint32 `protobuf:"varint,8,req,name=replica_placement" json:"replica_placement,omitempty"` |
|||
Version *uint32 `protobuf:"varint,9,opt,name=version,def=2" json:"version,omitempty"` |
|||
Ttl *uint32 `protobuf:"varint,10,opt,name=ttl" json:"ttl,omitempty"` |
|||
XXX_unrecognized []byte `json:"-"` |
|||
} |
|||
|
|||
func (m *VolumeInformationMessage) Reset() { *m = VolumeInformationMessage{} } |
|||
func (m *VolumeInformationMessage) String() string { return proto.CompactTextString(m) } |
|||
func (*VolumeInformationMessage) ProtoMessage() {} |
|||
|
|||
const Default_VolumeInformationMessage_Version uint32 = 2 |
|||
|
|||
func (m *VolumeInformationMessage) GetId() uint32 { |
|||
if m != nil && m.Id != nil { |
|||
return *m.Id |
|||
} |
|||
return 0 |
|||
} |
|||
|
|||
func (m *VolumeInformationMessage) GetSize() uint64 { |
|||
if m != nil && m.Size != nil { |
|||
return *m.Size |
|||
} |
|||
return 0 |
|||
} |
|||
|
|||
func (m *VolumeInformationMessage) GetCollection() string { |
|||
if m != nil && m.Collection != nil { |
|||
return *m.Collection |
|||
} |
|||
return "" |
|||
} |
|||
|
|||
func (m *VolumeInformationMessage) GetFileCount() uint64 { |
|||
if m != nil && m.FileCount != nil { |
|||
return *m.FileCount |
|||
} |
|||
return 0 |
|||
} |
|||
|
|||
func (m *VolumeInformationMessage) GetDeleteCount() uint64 { |
|||
if m != nil && m.DeleteCount != nil { |
|||
return *m.DeleteCount |
|||
} |
|||
return 0 |
|||
} |
|||
|
|||
func (m *VolumeInformationMessage) GetDeletedByteCount() uint64 { |
|||
if m != nil && m.DeletedByteCount != nil { |
|||
return *m.DeletedByteCount |
|||
} |
|||
return 0 |
|||
} |
|||
|
|||
func (m *VolumeInformationMessage) GetReadOnly() bool { |
|||
if m != nil && m.ReadOnly != nil { |
|||
return *m.ReadOnly |
|||
} |
|||
return false |
|||
} |
|||
|
|||
func (m *VolumeInformationMessage) GetReplicaPlacement() uint32 { |
|||
if m != nil && m.ReplicaPlacement != nil { |
|||
return *m.ReplicaPlacement |
|||
} |
|||
return 0 |
|||
} |
|||
|
|||
func (m *VolumeInformationMessage) GetVersion() uint32 { |
|||
if m != nil && m.Version != nil { |
|||
return *m.Version |
|||
} |
|||
return Default_VolumeInformationMessage_Version |
|||
} |
|||
|
|||
func (m *VolumeInformationMessage) GetTtl() uint32 { |
|||
if m != nil && m.Ttl != nil { |
|||
return *m.Ttl |
|||
} |
|||
return 0 |
|||
} |
|||
|
|||
type JoinMessage struct { |
|||
IsInit *bool `protobuf:"varint,1,opt,name=is_init" json:"is_init,omitempty"` |
|||
Ip *string `protobuf:"bytes,2,req,name=ip" json:"ip,omitempty"` |
|||
Port *uint32 `protobuf:"varint,3,req,name=port" json:"port,omitempty"` |
|||
PublicUrl *string `protobuf:"bytes,4,opt,name=public_url" json:"public_url,omitempty"` |
|||
MaxVolumeCount *uint32 `protobuf:"varint,5,req,name=max_volume_count" json:"max_volume_count,omitempty"` |
|||
MaxFileKey *uint64 `protobuf:"varint,6,req,name=max_file_key" json:"max_file_key,omitempty"` |
|||
DataCenter *string `protobuf:"bytes,7,opt,name=data_center" json:"data_center,omitempty"` |
|||
Rack *string `protobuf:"bytes,8,opt,name=rack" json:"rack,omitempty"` |
|||
Volumes []*VolumeInformationMessage `protobuf:"bytes,9,rep,name=volumes" json:"volumes,omitempty"` |
|||
AdminPort *uint32 `protobuf:"varint,10,opt,name=admin_port" json:"admin_port,omitempty"` |
|||
XXX_unrecognized []byte `json:"-"` |
|||
} |
|||
|
|||
func (m *JoinMessage) Reset() { *m = JoinMessage{} } |
|||
func (m *JoinMessage) String() string { return proto.CompactTextString(m) } |
|||
func (*JoinMessage) ProtoMessage() {} |
|||
|
|||
func (m *JoinMessage) GetIsInit() bool { |
|||
if m != nil && m.IsInit != nil { |
|||
return *m.IsInit |
|||
} |
|||
return false |
|||
} |
|||
|
|||
func (m *JoinMessage) GetIp() string { |
|||
if m != nil && m.Ip != nil { |
|||
return *m.Ip |
|||
} |
|||
return "" |
|||
} |
|||
|
|||
func (m *JoinMessage) GetPort() uint32 { |
|||
if m != nil && m.Port != nil { |
|||
return *m.Port |
|||
} |
|||
return 0 |
|||
} |
|||
|
|||
func (m *JoinMessage) GetPublicUrl() string { |
|||
if m != nil && m.PublicUrl != nil { |
|||
return *m.PublicUrl |
|||
} |
|||
return "" |
|||
} |
|||
|
|||
func (m *JoinMessage) GetMaxVolumeCount() uint32 { |
|||
if m != nil && m.MaxVolumeCount != nil { |
|||
return *m.MaxVolumeCount |
|||
} |
|||
return 0 |
|||
} |
|||
|
|||
func (m *JoinMessage) GetMaxFileKey() uint64 { |
|||
if m != nil && m.MaxFileKey != nil { |
|||
return *m.MaxFileKey |
|||
} |
|||
return 0 |
|||
} |
|||
|
|||
func (m *JoinMessage) GetDataCenter() string { |
|||
if m != nil && m.DataCenter != nil { |
|||
return *m.DataCenter |
|||
} |
|||
return "" |
|||
} |
|||
|
|||
func (m *JoinMessage) GetRack() string { |
|||
if m != nil && m.Rack != nil { |
|||
return *m.Rack |
|||
} |
|||
return "" |
|||
} |
|||
|
|||
func (m *JoinMessage) GetVolumes() []*VolumeInformationMessage { |
|||
if m != nil { |
|||
return m.Volumes |
|||
} |
|||
return nil |
|||
} |
|||
|
|||
func (m *JoinMessage) GetAdminPort() uint32 { |
|||
if m != nil && m.AdminPort != nil { |
|||
return *m.AdminPort |
|||
} |
|||
return 0 |
|||
} |
|||
|
|||
func init() { |
|||
} |
|||
@ -1,59 +0,0 @@ |
|||
package operation |
|||
|
|||
import ( |
|||
"encoding/json" |
|||
"log" |
|||
"testing" |
|||
|
|||
"github.com/golang/protobuf/proto" |
|||
) |
|||
|
|||
func TestSerialDeserial(t *testing.T) { |
|||
volumeMessage := &VolumeInformationMessage{ |
|||
Id: proto.Uint32(12), |
|||
Size: proto.Uint64(2341234), |
|||
Collection: proto.String("benchmark"), |
|||
FileCount: proto.Uint64(2341234), |
|||
DeleteCount: proto.Uint64(234), |
|||
DeletedByteCount: proto.Uint64(21234), |
|||
ReadOnly: proto.Bool(false), |
|||
ReplicaPlacement: proto.Uint32(210), |
|||
Version: proto.Uint32(2), |
|||
} |
|||
var volumeMessages []*VolumeInformationMessage |
|||
volumeMessages = append(volumeMessages, volumeMessage) |
|||
|
|||
joinMessage := &JoinMessage{ |
|||
IsInit: proto.Bool(true), |
|||
Ip: proto.String("127.0.3.12"), |
|||
Port: proto.Uint32(34546), |
|||
PublicUrl: proto.String("localhost:2342"), |
|||
MaxVolumeCount: proto.Uint32(210), |
|||
MaxFileKey: proto.Uint64(324234423), |
|||
DataCenter: proto.String("dc1"), |
|||
Rack: proto.String("rack2"), |
|||
Volumes: volumeMessages, |
|||
} |
|||
|
|||
data, err := proto.Marshal(joinMessage) |
|||
if err != nil { |
|||
log.Fatal("marshaling error: ", err) |
|||
} |
|||
newMessage := &JoinMessage{} |
|||
err = proto.Unmarshal(data, newMessage) |
|||
if err != nil { |
|||
log.Fatal("unmarshaling error: ", err) |
|||
} |
|||
log.Println("The pb data size is", len(data)) |
|||
|
|||
jsonData, jsonError := json.Marshal(joinMessage) |
|||
if jsonError != nil { |
|||
log.Fatal("json marshaling error: ", jsonError) |
|||
} |
|||
log.Println("The json data size is", len(jsonData), string(jsonData)) |
|||
|
|||
// Now test and newTest contain the same data.
|
|||
if *joinMessage.PublicUrl != *newMessage.PublicUrl { |
|||
log.Fatalf("data mismatch %q != %q", *joinMessage.PublicUrl, *newMessage.PublicUrl) |
|||
} |
|||
} |
|||
@ -0,0 +1,6 @@ |
|||
all: gen |
|||
|
|||
.PHONY : gen |
|||
|
|||
gen: |
|||
protoc seaweed.proto --go_out=plugins=grpc:. |
|||
@ -0,0 +1,384 @@ |
|||
// Code generated by protoc-gen-go.
|
|||
// source: seaweed.proto
|
|||
// DO NOT EDIT!
|
|||
|
|||
/* |
|||
Package pb is a generated protocol buffer package. |
|||
|
|||
It is generated from these files: |
|||
seaweed.proto |
|||
|
|||
It has these top-level messages: |
|||
Heartbeat |
|||
HeartbeatResponse |
|||
VolumeInformationMessage |
|||
*/ |
|||
package pb |
|||
|
|||
import proto "github.com/golang/protobuf/proto" |
|||
import fmt "fmt" |
|||
import math "math" |
|||
|
|||
import ( |
|||
context "golang.org/x/net/context" |
|||
grpc "google.golang.org/grpc" |
|||
) |
|||
|
|||
// Reference imports to suppress errors if they are not otherwise used.
|
|||
var _ = proto.Marshal |
|||
var _ = fmt.Errorf |
|||
var _ = math.Inf |
|||
|
|||
// This is a compile-time assertion to ensure that this generated file
|
|||
// is compatible with the proto package it is being compiled against.
|
|||
// A compilation error at this line likely means your copy of the
|
|||
// proto package needs to be updated.
|
|||
const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package
|
|||
|
|||
type Heartbeat struct { |
|||
IsInit bool `protobuf:"varint,1,opt,name=is_init,json=isInit" json:"is_init,omitempty"` |
|||
Ip string `protobuf:"bytes,2,opt,name=ip" json:"ip,omitempty"` |
|||
Port uint32 `protobuf:"varint,3,opt,name=port" json:"port,omitempty"` |
|||
PublicUrl string `protobuf:"bytes,4,opt,name=public_url,json=publicUrl" json:"public_url,omitempty"` |
|||
MaxVolumeCount uint32 `protobuf:"varint,5,opt,name=max_volume_count,json=maxVolumeCount" json:"max_volume_count,omitempty"` |
|||
MaxFileKey uint64 `protobuf:"varint,6,opt,name=max_file_key,json=maxFileKey" json:"max_file_key,omitempty"` |
|||
DataCenter string `protobuf:"bytes,7,opt,name=data_center,json=dataCenter" json:"data_center,omitempty"` |
|||
Rack string `protobuf:"bytes,8,opt,name=rack" json:"rack,omitempty"` |
|||
Volumes []*VolumeInformationMessage `protobuf:"bytes,9,rep,name=volumes" json:"volumes,omitempty"` |
|||
AdminPort uint32 `protobuf:"varint,10,opt,name=admin_port,json=adminPort" json:"admin_port,omitempty"` |
|||
} |
|||
|
|||
func (m *Heartbeat) Reset() { *m = Heartbeat{} } |
|||
func (m *Heartbeat) String() string { return proto.CompactTextString(m) } |
|||
func (*Heartbeat) ProtoMessage() {} |
|||
func (*Heartbeat) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{0} } |
|||
|
|||
func (m *Heartbeat) GetIsInit() bool { |
|||
if m != nil { |
|||
return m.IsInit |
|||
} |
|||
return false |
|||
} |
|||
|
|||
func (m *Heartbeat) GetIp() string { |
|||
if m != nil { |
|||
return m.Ip |
|||
} |
|||
return "" |
|||
} |
|||
|
|||
func (m *Heartbeat) GetPort() uint32 { |
|||
if m != nil { |
|||
return m.Port |
|||
} |
|||
return 0 |
|||
} |
|||
|
|||
func (m *Heartbeat) GetPublicUrl() string { |
|||
if m != nil { |
|||
return m.PublicUrl |
|||
} |
|||
return "" |
|||
} |
|||
|
|||
func (m *Heartbeat) GetMaxVolumeCount() uint32 { |
|||
if m != nil { |
|||
return m.MaxVolumeCount |
|||
} |
|||
return 0 |
|||
} |
|||
|
|||
func (m *Heartbeat) GetMaxFileKey() uint64 { |
|||
if m != nil { |
|||
return m.MaxFileKey |
|||
} |
|||
return 0 |
|||
} |
|||
|
|||
func (m *Heartbeat) GetDataCenter() string { |
|||
if m != nil { |
|||
return m.DataCenter |
|||
} |
|||
return "" |
|||
} |
|||
|
|||
func (m *Heartbeat) GetRack() string { |
|||
if m != nil { |
|||
return m.Rack |
|||
} |
|||
return "" |
|||
} |
|||
|
|||
func (m *Heartbeat) GetVolumes() []*VolumeInformationMessage { |
|||
if m != nil { |
|||
return m.Volumes |
|||
} |
|||
return nil |
|||
} |
|||
|
|||
func (m *Heartbeat) GetAdminPort() uint32 { |
|||
if m != nil { |
|||
return m.AdminPort |
|||
} |
|||
return 0 |
|||
} |
|||
|
|||
type HeartbeatResponse struct { |
|||
VolumeSizeLimit uint64 `protobuf:"varint,1,opt,name=volumeSizeLimit" json:"volumeSizeLimit,omitempty"` |
|||
SecretKey string `protobuf:"bytes,2,opt,name=secretKey" json:"secretKey,omitempty"` |
|||
} |
|||
|
|||
func (m *HeartbeatResponse) Reset() { *m = HeartbeatResponse{} } |
|||
func (m *HeartbeatResponse) String() string { return proto.CompactTextString(m) } |
|||
func (*HeartbeatResponse) ProtoMessage() {} |
|||
func (*HeartbeatResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{1} } |
|||
|
|||
func (m *HeartbeatResponse) GetVolumeSizeLimit() uint64 { |
|||
if m != nil { |
|||
return m.VolumeSizeLimit |
|||
} |
|||
return 0 |
|||
} |
|||
|
|||
func (m *HeartbeatResponse) GetSecretKey() string { |
|||
if m != nil { |
|||
return m.SecretKey |
|||
} |
|||
return "" |
|||
} |
|||
|
|||
type VolumeInformationMessage struct { |
|||
Id uint32 `protobuf:"varint,1,opt,name=id" json:"id,omitempty"` |
|||
Size uint64 `protobuf:"varint,2,opt,name=size" json:"size,omitempty"` |
|||
Collection string `protobuf:"bytes,3,opt,name=collection" json:"collection,omitempty"` |
|||
FileCount uint64 `protobuf:"varint,4,opt,name=file_count,json=fileCount" json:"file_count,omitempty"` |
|||
DeleteCount uint64 `protobuf:"varint,5,opt,name=delete_count,json=deleteCount" json:"delete_count,omitempty"` |
|||
DeletedByteCount uint64 `protobuf:"varint,6,opt,name=deleted_byte_count,json=deletedByteCount" json:"deleted_byte_count,omitempty"` |
|||
ReadOnly bool `protobuf:"varint,7,opt,name=read_only,json=readOnly" json:"read_only,omitempty"` |
|||
ReplicaPlacement uint32 `protobuf:"varint,8,opt,name=replica_placement,json=replicaPlacement" json:"replica_placement,omitempty"` |
|||
Version uint32 `protobuf:"varint,9,opt,name=version" json:"version,omitempty"` |
|||
Ttl uint32 `protobuf:"varint,10,opt,name=ttl" json:"ttl,omitempty"` |
|||
} |
|||
|
|||
func (m *VolumeInformationMessage) Reset() { *m = VolumeInformationMessage{} } |
|||
func (m *VolumeInformationMessage) String() string { return proto.CompactTextString(m) } |
|||
func (*VolumeInformationMessage) ProtoMessage() {} |
|||
func (*VolumeInformationMessage) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{2} } |
|||
|
|||
func (m *VolumeInformationMessage) GetId() uint32 { |
|||
if m != nil { |
|||
return m.Id |
|||
} |
|||
return 0 |
|||
} |
|||
|
|||
func (m *VolumeInformationMessage) GetSize() uint64 { |
|||
if m != nil { |
|||
return m.Size |
|||
} |
|||
return 0 |
|||
} |
|||
|
|||
func (m *VolumeInformationMessage) GetCollection() string { |
|||
if m != nil { |
|||
return m.Collection |
|||
} |
|||
return "" |
|||
} |
|||
|
|||
func (m *VolumeInformationMessage) GetFileCount() uint64 { |
|||
if m != nil { |
|||
return m.FileCount |
|||
} |
|||
return 0 |
|||
} |
|||
|
|||
func (m *VolumeInformationMessage) GetDeleteCount() uint64 { |
|||
if m != nil { |
|||
return m.DeleteCount |
|||
} |
|||
return 0 |
|||
} |
|||
|
|||
func (m *VolumeInformationMessage) GetDeletedByteCount() uint64 { |
|||
if m != nil { |
|||
return m.DeletedByteCount |
|||
} |
|||
return 0 |
|||
} |
|||
|
|||
func (m *VolumeInformationMessage) GetReadOnly() bool { |
|||
if m != nil { |
|||
return m.ReadOnly |
|||
} |
|||
return false |
|||
} |
|||
|
|||
func (m *VolumeInformationMessage) GetReplicaPlacement() uint32 { |
|||
if m != nil { |
|||
return m.ReplicaPlacement |
|||
} |
|||
return 0 |
|||
} |
|||
|
|||
func (m *VolumeInformationMessage) GetVersion() uint32 { |
|||
if m != nil { |
|||
return m.Version |
|||
} |
|||
return 0 |
|||
} |
|||
|
|||
func (m *VolumeInformationMessage) GetTtl() uint32 { |
|||
if m != nil { |
|||
return m.Ttl |
|||
} |
|||
return 0 |
|||
} |
|||
|
|||
func init() { |
|||
proto.RegisterType((*Heartbeat)(nil), "pb.Heartbeat") |
|||
proto.RegisterType((*HeartbeatResponse)(nil), "pb.HeartbeatResponse") |
|||
proto.RegisterType((*VolumeInformationMessage)(nil), "pb.VolumeInformationMessage") |
|||
} |
|||
|
|||
// Reference imports to suppress errors if they are not otherwise used.
|
|||
var _ context.Context |
|||
var _ grpc.ClientConn |
|||
|
|||
// This is a compile-time assertion to ensure that this generated file
|
|||
// is compatible with the grpc package it is being compiled against.
|
|||
const _ = grpc.SupportPackageIsVersion4 |
|||
|
|||
// Client API for Seaweed service
|
|||
|
|||
type SeaweedClient interface { |
|||
SendHeartbeat(ctx context.Context, opts ...grpc.CallOption) (Seaweed_SendHeartbeatClient, error) |
|||
} |
|||
|
|||
type seaweedClient struct { |
|||
cc *grpc.ClientConn |
|||
} |
|||
|
|||
func NewSeaweedClient(cc *grpc.ClientConn) SeaweedClient { |
|||
return &seaweedClient{cc} |
|||
} |
|||
|
|||
func (c *seaweedClient) SendHeartbeat(ctx context.Context, opts ...grpc.CallOption) (Seaweed_SendHeartbeatClient, error) { |
|||
stream, err := grpc.NewClientStream(ctx, &_Seaweed_serviceDesc.Streams[0], c.cc, "/pb.Seaweed/SendHeartbeat", opts...) |
|||
if err != nil { |
|||
return nil, err |
|||
} |
|||
x := &seaweedSendHeartbeatClient{stream} |
|||
return x, nil |
|||
} |
|||
|
|||
type Seaweed_SendHeartbeatClient interface { |
|||
Send(*Heartbeat) error |
|||
Recv() (*HeartbeatResponse, error) |
|||
grpc.ClientStream |
|||
} |
|||
|
|||
type seaweedSendHeartbeatClient struct { |
|||
grpc.ClientStream |
|||
} |
|||
|
|||
func (x *seaweedSendHeartbeatClient) Send(m *Heartbeat) error { |
|||
return x.ClientStream.SendMsg(m) |
|||
} |
|||
|
|||
func (x *seaweedSendHeartbeatClient) Recv() (*HeartbeatResponse, error) { |
|||
m := new(HeartbeatResponse) |
|||
if err := x.ClientStream.RecvMsg(m); err != nil { |
|||
return nil, err |
|||
} |
|||
return m, nil |
|||
} |
|||
|
|||
// Server API for Seaweed service
|
|||
|
|||
type SeaweedServer interface { |
|||
SendHeartbeat(Seaweed_SendHeartbeatServer) error |
|||
} |
|||
|
|||
func RegisterSeaweedServer(s *grpc.Server, srv SeaweedServer) { |
|||
s.RegisterService(&_Seaweed_serviceDesc, srv) |
|||
} |
|||
|
|||
func _Seaweed_SendHeartbeat_Handler(srv interface{}, stream grpc.ServerStream) error { |
|||
return srv.(SeaweedServer).SendHeartbeat(&seaweedSendHeartbeatServer{stream}) |
|||
} |
|||
|
|||
type Seaweed_SendHeartbeatServer interface { |
|||
Send(*HeartbeatResponse) error |
|||
Recv() (*Heartbeat, error) |
|||
grpc.ServerStream |
|||
} |
|||
|
|||
type seaweedSendHeartbeatServer struct { |
|||
grpc.ServerStream |
|||
} |
|||
|
|||
func (x *seaweedSendHeartbeatServer) Send(m *HeartbeatResponse) error { |
|||
return x.ServerStream.SendMsg(m) |
|||
} |
|||
|
|||
func (x *seaweedSendHeartbeatServer) Recv() (*Heartbeat, error) { |
|||
m := new(Heartbeat) |
|||
if err := x.ServerStream.RecvMsg(m); err != nil { |
|||
return nil, err |
|||
} |
|||
return m, nil |
|||
} |
|||
|
|||
var _Seaweed_serviceDesc = grpc.ServiceDesc{ |
|||
ServiceName: "pb.Seaweed", |
|||
HandlerType: (*SeaweedServer)(nil), |
|||
Methods: []grpc.MethodDesc{}, |
|||
Streams: []grpc.StreamDesc{ |
|||
{ |
|||
StreamName: "SendHeartbeat", |
|||
Handler: _Seaweed_SendHeartbeat_Handler, |
|||
ServerStreams: true, |
|||
ClientStreams: true, |
|||
}, |
|||
}, |
|||
Metadata: "seaweed.proto", |
|||
} |
|||
|
|||
func init() { proto.RegisterFile("seaweed.proto", fileDescriptor0) } |
|||
|
|||
var fileDescriptor0 = []byte{ |
|||
// 511 bytes of a gzipped FileDescriptorProto
|
|||
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0x74, 0x93, 0x41, 0x6f, 0xd3, 0x4c, |
|||
0x10, 0x86, 0x3f, 0x3b, 0xfe, 0x92, 0x78, 0x52, 0x97, 0x74, 0x25, 0x84, 0x05, 0x05, 0x4c, 0x4e, |
|||
0x96, 0x40, 0x11, 0x2a, 0x12, 0x17, 0x6e, 0x54, 0xaa, 0xa8, 0x0a, 0xa2, 0xda, 0x08, 0x2e, 0x1c, |
|||
0xac, 0xb5, 0x3d, 0x45, 0xab, 0xae, 0xd7, 0xd6, 0x7a, 0x53, 0xe2, 0xfe, 0x39, 0x2e, 0xfc, 0x30, |
|||
0xb4, 0xb3, 0x49, 0x5a, 0x90, 0xb8, 0xcd, 0x3c, 0xfb, 0x8e, 0x77, 0x67, 0xde, 0x31, 0x24, 0x3d, |
|||
0x8a, 0x1f, 0x88, 0xf5, 0xb2, 0x33, 0xad, 0x6d, 0x59, 0xd8, 0x95, 0x8b, 0x9f, 0x21, 0xc4, 0x1f, |
|||
0x50, 0x18, 0x5b, 0xa2, 0xb0, 0xec, 0x11, 0x4c, 0x64, 0x5f, 0x48, 0x2d, 0x6d, 0x1a, 0x64, 0x41, |
|||
0x3e, 0xe5, 0x63, 0xd9, 0x9f, 0x6b, 0x69, 0xd9, 0x21, 0x84, 0xb2, 0x4b, 0xc3, 0x2c, 0xc8, 0x63, |
|||
0x1e, 0xca, 0x8e, 0x31, 0x88, 0xba, 0xd6, 0xd8, 0x74, 0x94, 0x05, 0x79, 0xc2, 0x29, 0x66, 0x4f, |
|||
0x01, 0xba, 0x75, 0xa9, 0x64, 0x55, 0xac, 0x8d, 0x4a, 0x23, 0xd2, 0xc6, 0x9e, 0x7c, 0x31, 0x8a, |
|||
0xe5, 0x30, 0x6f, 0xc4, 0xa6, 0xb8, 0x69, 0xd5, 0xba, 0xc1, 0xa2, 0x6a, 0xd7, 0xda, 0xa6, 0xff, |
|||
0x53, 0xf9, 0x61, 0x23, 0x36, 0x5f, 0x09, 0x9f, 0x3a, 0xca, 0x32, 0x38, 0x70, 0xca, 0x2b, 0xa9, |
|||
0xb0, 0xb8, 0xc6, 0x21, 0x1d, 0x67, 0x41, 0x1e, 0x71, 0x68, 0xc4, 0xe6, 0x4c, 0x2a, 0xbc, 0xc0, |
|||
0x81, 0x3d, 0x87, 0x59, 0x2d, 0xac, 0x28, 0x2a, 0xd4, 0x16, 0x4d, 0x3a, 0xa1, 0xbb, 0xc0, 0xa1, |
|||
0x53, 0x22, 0xee, 0x7d, 0x46, 0x54, 0xd7, 0xe9, 0x94, 0x4e, 0x28, 0x66, 0x6f, 0x61, 0xe2, 0x2f, |
|||
0xef, 0xd3, 0x38, 0x1b, 0xe5, 0xb3, 0x93, 0xe3, 0x65, 0x57, 0x2e, 0xfd, 0xc5, 0xe7, 0xfa, 0xaa, |
|||
0x35, 0x8d, 0xb0, 0xb2, 0xd5, 0x9f, 0xb0, 0xef, 0xc5, 0x77, 0xe4, 0x3b, 0xb1, 0xeb, 0x4b, 0xd4, |
|||
0x8d, 0xd4, 0x05, 0x75, 0x0c, 0xf4, 0xe4, 0x98, 0xc8, 0x65, 0x6b, 0xec, 0xe2, 0x1b, 0x1c, 0xed, |
|||
0x07, 0xc8, 0xb1, 0xef, 0x5a, 0xdd, 0x23, 0xcb, 0xe1, 0x81, 0x2f, 0x5f, 0xc9, 0x5b, 0xfc, 0x28, |
|||
0x9b, 0xed, 0x40, 0x23, 0xfe, 0x37, 0x66, 0xc7, 0x10, 0xf7, 0x58, 0x19, 0xb4, 0x17, 0x38, 0x6c, |
|||
0x07, 0x7c, 0x07, 0x16, 0xbf, 0x42, 0x48, 0xff, 0xf5, 0x42, 0x32, 0xa5, 0xa6, 0xef, 0x26, 0x3c, |
|||
0x94, 0xb5, 0x6b, 0xba, 0x97, 0xb7, 0x48, 0x5f, 0x89, 0x38, 0xc5, 0xec, 0x19, 0x40, 0xd5, 0x2a, |
|||
0x85, 0x95, 0x2b, 0x24, 0xbb, 0x62, 0x7e, 0x8f, 0xb8, 0xe6, 0x68, 0xce, 0xde, 0x8f, 0x88, 0x2a, |
|||
0x63, 0x47, 0xbc, 0x15, 0x2f, 0xe0, 0xa0, 0x46, 0x85, 0xf6, 0xbe, 0x61, 0x11, 0x9f, 0x79, 0xe6, |
|||
0x25, 0xaf, 0x80, 0xf9, 0xb4, 0x2e, 0xca, 0x61, 0x2f, 0xf4, 0x9e, 0xcd, 0xb7, 0x27, 0xef, 0x87, |
|||
0x9d, 0xfa, 0x09, 0xc4, 0x06, 0x45, 0x5d, 0xb4, 0x5a, 0x0d, 0xe4, 0xdb, 0x94, 0x4f, 0x1d, 0xf8, |
|||
0xac, 0xd5, 0xc0, 0x5e, 0xc2, 0x91, 0xc1, 0x4e, 0xc9, 0x4a, 0x14, 0x9d, 0x12, 0x15, 0x36, 0xa8, |
|||
0x2d, 0x59, 0x98, 0xf0, 0xf9, 0xf6, 0xe0, 0x72, 0xc7, 0x59, 0x0a, 0x93, 0x1b, 0x34, 0xbd, 0x6b, |
|||
0x2b, 0x26, 0xc9, 0x2e, 0x65, 0x73, 0x18, 0x59, 0xab, 0xb6, 0x4e, 0xb9, 0xf0, 0xe4, 0x0c, 0x26, |
|||
0x2b, 0xbf, 0xfa, 0xec, 0x1d, 0x24, 0x2b, 0xd4, 0xf5, 0xdd, 0xce, 0x27, 0x6e, 0x0b, 0xf6, 0xe9, |
|||
0xe3, 0x87, 0x7f, 0xa4, 0x3b, 0x43, 0x17, 0xff, 0xe5, 0xc1, 0xeb, 0xa0, 0x1c, 0xd3, 0x8f, 0xf3, |
|||
0xe6, 0x77, 0x00, 0x00, 0x00, 0xff, 0xff, 0x7d, 0xc1, 0xd3, 0x35, 0x49, 0x03, 0x00, 0x00, |
|||
} |
|||
@ -0,0 +1,41 @@ |
|||
syntax = "proto3"; |
|||
|
|||
package pb; |
|||
|
|||
////////////////////////////////////////////////// |
|||
|
|||
service Seaweed { |
|||
rpc SendHeartbeat(stream Heartbeat) returns (stream HeartbeatResponse) {} |
|||
} |
|||
|
|||
////////////////////////////////////////////////// |
|||
|
|||
message Heartbeat { |
|||
bool is_init = 1; |
|||
string ip = 2; |
|||
uint32 port = 3; |
|||
string public_url = 4; |
|||
uint32 max_volume_count = 5; |
|||
uint64 max_file_key = 6; |
|||
string data_center = 7; |
|||
string rack = 8; |
|||
repeated VolumeInformationMessage volumes = 9; |
|||
uint32 admin_port = 10; |
|||
} |
|||
message HeartbeatResponse { |
|||
uint64 volumeSizeLimit = 1; |
|||
string secretKey = 2; |
|||
} |
|||
|
|||
message VolumeInformationMessage { |
|||
uint32 id = 1; |
|||
uint64 size = 2; |
|||
string collection = 3; |
|||
uint64 file_count = 4; |
|||
uint64 delete_count = 5; |
|||
uint64 deleted_byte_count = 6; |
|||
bool read_only = 7; |
|||
uint32 replica_placement = 8; |
|||
uint32 version = 9; |
|||
uint32 ttl = 10; |
|||
} |
|||
@ -0,0 +1,57 @@ |
|||
package weed_server |
|||
|
|||
import ( |
|||
"github.com/chrislusf/seaweedfs/weed/glog" |
|||
"github.com/chrislusf/seaweedfs/weed/pb" |
|||
"github.com/chrislusf/seaweedfs/weed/storage" |
|||
"github.com/chrislusf/seaweedfs/weed/topology" |
|||
) |
|||
|
|||
func (ms MasterServer) SendHeartbeat(stream pb.Seaweed_SendHeartbeatServer) error { |
|||
var dn *topology.DataNode |
|||
t := ms.Topo |
|||
for { |
|||
heartbeat, err := stream.Recv() |
|||
if err == nil { |
|||
if dn == nil { |
|||
t.Sequence.SetMax(heartbeat.MaxFileKey) |
|||
dcName, rackName := t.Configuration.Locate(heartbeat.Ip, heartbeat.DataCenter, heartbeat.Rack) |
|||
dc := t.GetOrCreateDataCenter(dcName) |
|||
rack := dc.GetOrCreateRack(rackName) |
|||
dn = rack.GetOrCreateDataNode(heartbeat.Ip, |
|||
int(heartbeat.Port), heartbeat.PublicUrl, |
|||
int(heartbeat.MaxVolumeCount)) |
|||
glog.V(0).Infof("added volume server %v:%d", heartbeat.GetIp(), heartbeat.GetPort()) |
|||
if err := stream.Send(&pb.HeartbeatResponse{ |
|||
VolumeSizeLimit: uint64(ms.volumeSizeLimitMB) * 1024 * 1024, |
|||
SecretKey: string(ms.guard.SecretKey), |
|||
}); err != nil { |
|||
return err |
|||
} |
|||
} |
|||
|
|||
var volumeInfos []storage.VolumeInfo |
|||
for _, v := range heartbeat.Volumes { |
|||
if vi, err := storage.NewVolumeInfo(v); err == nil { |
|||
volumeInfos = append(volumeInfos, vi) |
|||
} else { |
|||
glog.V(0).Infof("Fail to convert joined volume information: %v", err) |
|||
} |
|||
} |
|||
deletedVolumes := dn.UpdateVolumes(volumeInfos) |
|||
for _, v := range volumeInfos { |
|||
t.RegisterVolumeLayout(v, dn) |
|||
} |
|||
for _, v := range deletedVolumes { |
|||
t.UnRegisterVolumeLayout(v, dn) |
|||
} |
|||
|
|||
} else { |
|||
glog.V(0).Infof("lost volume server %s:%d", dn.Ip, dn.Port) |
|||
if dn != nil { |
|||
t.UnRegisterDataNode(dn) |
|||
} |
|||
return err |
|||
} |
|||
} |
|||
} |
|||
@ -0,0 +1,74 @@ |
|||
package weed_server |
|||
|
|||
import ( |
|||
"fmt" |
|||
"time" |
|||
|
|||
"github.com/chrislusf/seaweedfs/weed/glog" |
|||
"github.com/chrislusf/seaweedfs/weed/pb" |
|||
"github.com/chrislusf/seaweedfs/weed/security" |
|||
"github.com/chrislusf/seaweedfs/weed/storage" |
|||
"golang.org/x/net/context" |
|||
"google.golang.org/grpc" |
|||
) |
|||
|
|||
func (vs *VolumeServer) heartbeat() { |
|||
|
|||
glog.V(0).Infof("Volume server bootstraps with master %s", vs.GetMasterNode()) |
|||
vs.masterNodes = storage.NewMasterNodes(vs.masterNode) |
|||
vs.store.SetDataCenter(vs.dataCenter) |
|||
vs.store.SetRack(vs.rack) |
|||
|
|||
for { |
|||
err := vs.doHeartbeat(time.Duration(vs.pulseSeconds) * time.Second) |
|||
if err != nil { |
|||
glog.V(0).Infof("heartbeat error: %v", err) |
|||
time.Sleep(time.Duration(3*vs.pulseSeconds) * time.Second) |
|||
} |
|||
} |
|||
} |
|||
|
|||
func (vs *VolumeServer) doHeartbeat(sleepInterval time.Duration) error { |
|||
|
|||
masterNode, err := vs.masterNodes.FindMaster() |
|||
if err != nil { |
|||
return fmt.Errorf("No master found: %v", err) |
|||
} |
|||
|
|||
grpcConection, err := grpc.Dial(masterNode, grpc.WithInsecure()) |
|||
if err != nil { |
|||
return fmt.Errorf("fail to dial: %v", err) |
|||
} |
|||
defer grpcConection.Close() |
|||
|
|||
client := pb.NewSeaweedClient(grpcConection) |
|||
stream, err := client.SendHeartbeat(context.Background()) |
|||
if err != nil { |
|||
glog.V(0).Infof("%v.SendHeartbeat(_) = _, %v", client, err) |
|||
return err |
|||
} |
|||
vs.SetMasterNode(masterNode) |
|||
glog.V(0).Infof("Heartbeat to %s", masterNode) |
|||
|
|||
vs.store.Client = stream |
|||
defer func() { vs.store.Client = nil }() |
|||
|
|||
go func() { |
|||
for { |
|||
in, err := stream.Recv() |
|||
if err != nil { |
|||
return |
|||
} |
|||
vs.store.VolumeSizeLimit = in.GetVolumeSizeLimit() |
|||
vs.guard.SecretKey = security.Secret(in.GetSecretKey()) |
|||
} |
|||
}() |
|||
|
|||
for { |
|||
if err = stream.Send(vs.store.CollectHeartbeat()); err != nil { |
|||
glog.V(0).Infof("Volume Server Failed to talk with master %s: %v", masterNode, err) |
|||
return err |
|||
} |
|||
time.Sleep(sleepInterval) |
|||
} |
|||
} |
|||
Write
Preview
Loading…
Cancel
Save
Reference in new issue