From 1e3e4b3072071341b4bb4b0bb7c611457e927f97 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Tue, 5 May 2020 02:05:28 -0700 Subject: [PATCH 01/24] add broker connects to filer --- other/java/client/src/main/proto/filer.proto | 9 + weed/command/msg_broker.go | 8 +- .../broker/broker_grpc_server_discovery.go | 87 +++++ .../broker/broker_grpc_server_subscribe.go | 1 - weed/messaging/broker/broker_server.go | 76 ++-- weed/messaging/client/client.go | 30 -- weed/messaging/msgclient/client.go | 56 +++ weed/messaging/msgclient/pub_sub_chan.go | 96 +++++ .../{client => msgclient}/publisher.go | 14 +- .../{client => msgclient}/subscriber.go | 25 +- weed/pb/filer.proto | 9 + weed/pb/filer_pb/filer.pb.go | 353 ++++++++++++------ weed/pb/messaging.proto | 18 +- weed/pb/messaging_pb/messaging.pb.go | 237 +++++++----- weed/pb/volume_server_pb/volume_server.pb.go | 26 +- weed/server/filer_grpc_server.go | 36 ++ weed/server/filer_server.go | 7 +- 17 files changed, 761 insertions(+), 327 deletions(-) create mode 100644 weed/messaging/broker/broker_grpc_server_discovery.go delete mode 100644 weed/messaging/client/client.go create mode 100644 weed/messaging/msgclient/client.go create mode 100644 weed/messaging/msgclient/pub_sub_chan.go rename weed/messaging/{client => msgclient}/publisher.go (84%) rename weed/messaging/{client => msgclient}/subscriber.go (79%) diff --git a/other/java/client/src/main/proto/filer.proto b/other/java/client/src/main/proto/filer.proto index 6d890861d..3b3b78bbb 100644 --- a/other/java/client/src/main/proto/filer.proto +++ b/other/java/client/src/main/proto/filer.proto @@ -48,6 +48,8 @@ service SeaweedFiler { rpc SubscribeMetadata (SubscribeMetadataRequest) returns (stream SubscribeMetadataResponse) { } + rpc KeepConnected (stream KeepConnectedRequest) returns (stream KeepConnectedResponse) { + } } ////////////////////////////////////////////////// @@ -261,3 +263,10 @@ message LogEntry { int32 partition_key_hash = 2; bytes data = 3; } + +message KeepConnectedRequest { + string name = 1; + uint32 grpc_port = 2; +} +message KeepConnectedResponse { +} diff --git a/weed/command/msg_broker.go b/weed/command/msg_broker.go index 984497c2e..bab1083ab 100644 --- a/weed/command/msg_broker.go +++ b/weed/command/msg_broker.go @@ -6,9 +6,10 @@ import ( "strconv" "time" - "github.com/chrislusf/seaweedfs/weed/util/grace" "google.golang.org/grpc/reflection" + "github.com/chrislusf/seaweedfs/weed/util/grace" + "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/messaging/broker" "github.com/chrislusf/seaweedfs/weed/pb" @@ -24,6 +25,7 @@ var ( type QueueOptions struct { filer *string + ip *string port *int cpuprofile *string memprofile *string @@ -32,7 +34,8 @@ type QueueOptions struct { func init() { cmdMsgBroker.Run = runMsgBroker // break init cycle messageBrokerStandaloneOptions.filer = cmdMsgBroker.Flag.String("filer", "localhost:8888", "filer server address") - messageBrokerStandaloneOptions.port = cmdMsgBroker.Flag.Int("port", 17777, "queue server gRPC listen port") + messageBrokerStandaloneOptions.ip = cmdMsgBroker.Flag.String("ip", util.DetectedHostAddress(), "broker host address") + messageBrokerStandaloneOptions.port = cmdMsgBroker.Flag.Int("port", 17777, "broker gRPC listen port") messageBrokerStandaloneOptions.cpuprofile = cmdMsgBroker.Flag.String("cpuprofile", "", "cpu profile output file") messageBrokerStandaloneOptions.memprofile = cmdMsgBroker.Flag.String("memprofile", "", "memory profile output file") } @@ -91,6 +94,7 @@ func (msgBrokerOpt *QueueOptions) startQueueServer() bool { Filers: []string{*msgBrokerOpt.filer}, DefaultReplication: "", MaxMB: 0, + Ip: *msgBrokerOpt.ip, Port: *msgBrokerOpt.port, Cipher: cipher, }, grpcDialOption) diff --git a/weed/messaging/broker/broker_grpc_server_discovery.go b/weed/messaging/broker/broker_grpc_server_discovery.go new file mode 100644 index 000000000..4b7f357fa --- /dev/null +++ b/weed/messaging/broker/broker_grpc_server_discovery.go @@ -0,0 +1,87 @@ +package broker + +import ( + "context" + "time" + + "github.com/chrislusf/seaweedfs/weed/glog" + "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" + "github.com/chrislusf/seaweedfs/weed/pb/master_pb" + "github.com/chrislusf/seaweedfs/weed/pb/messaging_pb" +) + +/* +Topic discovery: + +When pub or sub connects, it ask for the whole broker list, and run consistent hashing to find the broker. + +The broker will check peers whether it is already hosted by some other broker, if that broker is alive and acknowledged alive, redirect to it. +Otherwise, just host the topic. + +So, if the pub or sub connects around the same time, they would connect to the same broker. Everyone is happy. +If one of the pub or sub connects very late, and the system topo changed quite a bit with new servers added or old servers died, checking peers will help. + +*/ + +func (broker *MessageBroker) FindBroker(c context.Context, request *messaging_pb.FindBrokerRequest) (*messaging_pb.FindBrokerResponse, error) { + + panic("implement me") +} + + + +func (broker *MessageBroker) checkPeers() { + + // contact a filer about masters + var masters []string + found := false + for !found { + for _, filer := range broker.option.Filers { + err := broker.withFilerClient(filer, func(client filer_pb.SeaweedFilerClient) error { + resp, err := client.GetFilerConfiguration(context.Background(), &filer_pb.GetFilerConfigurationRequest{}) + if err != nil { + return err + } + masters = append(masters, resp.Masters...) + return nil + }) + if err == nil { + found = true + break + } + glog.V(0).Infof("failed to read masters from %+v: %v", broker.option.Filers, err) + time.Sleep(time.Second) + } + } + glog.V(0).Infof("received master list: %s", masters) + + // contact each masters for filers + var filers []string + found = false + for !found { + for _, master := range masters { + err := broker.withMasterClient(master, func(client master_pb.SeaweedClient) error { + resp, err := client.ListMasterClients(context.Background(), &master_pb.ListMasterClientsRequest{ + ClientType: "filer", + }) + if err != nil { + return err + } + + filers = append(filers, resp.GrpcAddresses...) + + return nil + }) + if err == nil { + found = true + break + } + glog.V(0).Infof("failed to list filers: %v", err) + time.Sleep(time.Second) + } + } + glog.V(0).Infof("received filer list: %s", filers) + + broker.option.Filers = filers + +} diff --git a/weed/messaging/broker/broker_grpc_server_subscribe.go b/weed/messaging/broker/broker_grpc_server_subscribe.go index 379063eed..472a5007b 100644 --- a/weed/messaging/broker/broker_grpc_server_subscribe.go +++ b/weed/messaging/broker/broker_grpc_server_subscribe.go @@ -38,7 +38,6 @@ func (broker *MessageBroker) Subscribe(stream messaging_pb.SeaweedMessaging_Subs } if err = stream.Send(&messaging_pb.BrokerMessage{ - Redirect: nil, }); err != nil { return err } diff --git a/weed/messaging/broker/broker_server.go b/weed/messaging/broker/broker_server.go index 29c227274..9cad27214 100644 --- a/weed/messaging/broker/broker_server.go +++ b/weed/messaging/broker/broker_server.go @@ -16,6 +16,7 @@ type MessageBrokerOption struct { Filers []string DefaultReplication string MaxMB int + Ip string Port int Cipher bool } @@ -37,73 +38,44 @@ func NewMessageBroker(option *MessageBrokerOption, grpcDialOption grpc.DialOptio messageBroker.checkPeers() - // go messageBroker.loopForEver() + go messageBroker.keepConnectedToOneFiler() return messageBroker, nil } -func (broker *MessageBroker) loopForEver() { +func (broker *MessageBroker) keepConnectedToOneFiler() { for { - broker.checkPeers() - time.Sleep(3 * time.Second) - } - -} - -func (broker *MessageBroker) checkPeers() { - - // contact a filer about masters - var masters []string - found := false - for !found { for _, filer := range broker.option.Filers { - err := broker.withFilerClient(filer, func(client filer_pb.SeaweedFilerClient) error { - resp, err := client.GetFilerConfiguration(context.Background(), &filer_pb.GetFilerConfigurationRequest{}) + broker.withFilerClient(filer, func(client filer_pb.SeaweedFilerClient) error { + stream, err := client.KeepConnected(context.Background()) if err != nil { + glog.V(0).Infof("%s:%d failed to keep connected to %s: %v", broker.option.Ip, broker.option.Port, filer, err) return err } - masters = append(masters, resp.Masters...) - return nil - }) - if err == nil { - found = true - break - } - glog.V(0).Infof("failed to read masters from %+v: %v", broker.option.Filers, err) - time.Sleep(time.Second) - } - } - glog.V(0).Infof("received master list: %s", masters) - - // contact each masters for filers - var filers []string - found = false - for !found { - for _, master := range masters { - err := broker.withMasterClient(master, func(client master_pb.SeaweedClient) error { - resp, err := client.ListMasterClients(context.Background(), &master_pb.ListMasterClientsRequest{ - ClientType: "filer", - }) - if err != nil { - return err + glog.V(0).Infof("conntected with filer: %v", filer) + for { + if err := stream.Send(&filer_pb.KeepConnectedRequest{ + Name: broker.option.Ip, + GrpcPort: uint32(broker.option.Port), + }); err != nil { + glog.V(0).Infof("%s:%d failed to sendto %s: %v", broker.option.Ip, broker.option.Port, filer, err) + return err + } + // println("send heartbeat") + if _, err := stream.Recv(); err != nil { + glog.V(0).Infof("%s:%d failed to receive from %s: %v", broker.option.Ip, broker.option.Port, filer, err) + return err + } + // println("received reply") + time.Sleep(11*time.Second) + // println("woke up") } - - filers = append(filers, resp.GrpcAddresses...) - return nil }) - if err == nil { - found = true - break - } - glog.V(0).Infof("failed to list filers: %v", err) - time.Sleep(time.Second) + time.Sleep(3*time.Second) } } - glog.V(0).Infof("received filer list: %s", filers) - - broker.option.Filers = filers } diff --git a/weed/messaging/client/client.go b/weed/messaging/client/client.go deleted file mode 100644 index 4a674a9fc..000000000 --- a/weed/messaging/client/client.go +++ /dev/null @@ -1,30 +0,0 @@ -package client - -import ( - "context" - - "google.golang.org/grpc" - - "github.com/chrislusf/seaweedfs/weed/pb" - "github.com/chrislusf/seaweedfs/weed/security" - "github.com/chrislusf/seaweedfs/weed/util" -) - -type MessagingClient struct { - bootstrapBrokers []string - grpcConnection *grpc.ClientConn -} - -func NewMessagingClient(bootstrapBrokers []string) (*MessagingClient, error) { - grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.msg_client") - - grpcConnection, err := pb.GrpcDial(context.Background(), "localhost:17777", grpcDialOption) - if err != nil { - return nil, err - } - - return &MessagingClient{ - bootstrapBrokers: bootstrapBrokers, - grpcConnection: grpcConnection, - }, nil -} diff --git a/weed/messaging/msgclient/client.go b/weed/messaging/msgclient/client.go new file mode 100644 index 000000000..f4e11232e --- /dev/null +++ b/weed/messaging/msgclient/client.go @@ -0,0 +1,56 @@ +package msgclient + +import ( + "context" + "fmt" + "log" + + "google.golang.org/grpc" + + "github.com/chrislusf/seaweedfs/weed/messaging/broker" + "github.com/chrislusf/seaweedfs/weed/pb" + "github.com/chrislusf/seaweedfs/weed/pb/messaging_pb" + "github.com/chrislusf/seaweedfs/weed/security" + "github.com/chrislusf/seaweedfs/weed/util" +) + +type MessagingClient struct { + bootstrapBrokers []string + grpcConnections map[broker.TopicPartition]*grpc.ClientConn + grpcDialOption grpc.DialOption +} + +func NewMessagingClient(bootstrapBrokers ...string) *MessagingClient { + return &MessagingClient{ + bootstrapBrokers: bootstrapBrokers, + grpcConnections: make(map[broker.TopicPartition]*grpc.ClientConn), + grpcDialOption: security.LoadClientTLS(util.GetViper(), "grpc.msg_client"), + } +} + + +func (mc *MessagingClient) findBroker(tp broker.TopicPartition) (*grpc.ClientConn, error) { + + for _, broker := range mc.bootstrapBrokers { + grpcConnection, err := pb.GrpcDial(context.Background(), broker, mc.grpcDialOption) + if err != nil { + log.Printf("dial broker %s: %v", broker, err) + continue + } + defer grpcConnection.Close() + + resp, err := messaging_pb.NewSeaweedMessagingClient(grpcConnection).FindBroker(context.Background(), + &messaging_pb.FindBrokerRequest{ + Namespace: tp.Namespace, + Topic: tp.Topic, + Parition: tp.Partition, + }) + if err != nil { + return nil, err + } + + targetBroker := resp.Broker + return pb.GrpcDial(context.Background(), targetBroker, mc.grpcDialOption) + } + return nil, fmt.Errorf("no broker found for %+v", tp) +} diff --git a/weed/messaging/msgclient/pub_sub_chan.go b/weed/messaging/msgclient/pub_sub_chan.go new file mode 100644 index 000000000..d39e4c658 --- /dev/null +++ b/weed/messaging/msgclient/pub_sub_chan.go @@ -0,0 +1,96 @@ +package msgclient + +import ( + "io" + "log" + "time" + + "github.com/chrislusf/seaweedfs/weed/messaging/broker" + "github.com/chrislusf/seaweedfs/weed/pb/messaging_pb" +) + +type PubChannel struct { + client messaging_pb.SeaweedMessaging_PublishClient +} + +func (mc *MessagingClient) NewPubChannel(chanName string) (*PubChannel, error) { + tp := broker.TopicPartition{ + Namespace: "chan", + Topic: chanName, + Partition: 0, + } + grpcConnection, err := mc.findBroker(tp) + if err != nil { + return nil, err + } + pc, err := setupPublisherClient(grpcConnection, tp) + if err != nil { + return nil, err + } + return &PubChannel{ + client: pc, + }, nil +} + +func (pc *PubChannel) Publish(m []byte) error { + return pc.client.Send(&messaging_pb.PublishRequest{ + Data: &messaging_pb.Message{ + Value: m, + }, + }) +} +func (pc *PubChannel) Close() error { + return pc.client.CloseSend() +} + +type SubChannel struct { + ch chan []byte + stream messaging_pb.SeaweedMessaging_SubscribeClient +} + +func (mc *MessagingClient) NewSubChannel(chanName string) (*SubChannel, error) { + tp := broker.TopicPartition{ + Namespace: "chan", + Topic: chanName, + Partition: 0, + } + grpcConnection, err := mc.findBroker(tp) + if err != nil { + return nil, err + } + sc, err := setupSubscriberClient(grpcConnection, "", "chan", chanName, 0, time.Unix(0,0)) + if err != nil { + return nil, err + } + + t := &SubChannel{ + ch: make(chan []byte), + stream: sc, + } + + go func() { + for { + resp, subErr := t.stream.Recv() + if subErr == io.EOF { + return + } + if subErr != nil { + log.Printf("fail to receive from netchan %s: %v", chanName, subErr) + return + } + if resp.IsClose { + close(t.ch) + return + } + if resp.Data != nil { + t.ch <- resp.Data.Value + } + } + }() + + return t, nil +} + +func (sc *SubChannel) Channel() chan []byte { + return sc.ch +} diff --git a/weed/messaging/client/publisher.go b/weed/messaging/msgclient/publisher.go similarity index 84% rename from weed/messaging/client/publisher.go rename to weed/messaging/msgclient/publisher.go index 68e5729c1..b0459494b 100644 --- a/weed/messaging/client/publisher.go +++ b/weed/messaging/msgclient/publisher.go @@ -1,10 +1,12 @@ -package client +package msgclient import ( "context" "github.com/OneOfOne/xxhash" + "github.com/chrislusf/seaweedfs/weed/messaging/broker" + "github.com/chrislusf/seaweedfs/weed/pb" "github.com/chrislusf/seaweedfs/weed/pb/messaging_pb" ) @@ -34,9 +36,9 @@ func (mc *MessagingClient) NewPublisher(publisherId, namespace, topic string) (* }, nil } -func (mc *MessagingClient) setupPublisherClient(namespace, topic string, partition int32) (messaging_pb.SeaweedMessaging_PublishClient, error) { +func setupPublisherClient(grpcConnection *grpc.ClientConn, tp broker.TopicPartition) (messaging_pb.SeaweedMessaging_PublishClient, error) { - stream, err := messaging_pb.NewSeaweedMessagingClient(mc.grpcConnection).Publish(context.Background()) + stream, err := messaging_pb.NewSeaweedMessagingClient(grpcConnection).Publish(context.Background()) if err != nil { return nil, err } @@ -44,9 +46,9 @@ func (mc *MessagingClient) setupPublisherClient(namespace, topic string, partiti // send init message err = stream.Send(&messaging_pb.PublishRequest{ Init: &messaging_pb.PublishRequest_InitMessage{ - Namespace: namespace, - Topic: topic, - Partition: partition, + Namespace: tp.Namespace, + Topic: tp.Topic, + Partition: tp.Partition, }, }) if err != nil { diff --git a/weed/messaging/client/subscriber.go b/weed/messaging/msgclient/subscriber.go similarity index 79% rename from weed/messaging/client/subscriber.go rename to weed/messaging/msgclient/subscriber.go index 53e7ffc7d..27fa35a5b 100644 --- a/weed/messaging/client/subscriber.go +++ b/weed/messaging/msgclient/subscriber.go @@ -1,4 +1,4 @@ -package client +package msgclient import ( "context" @@ -36,9 +36,22 @@ func (mc *MessagingClient) NewSubscriber(subscriberId, namespace, topic string, func (mc *MessagingClient) setupSubscriberClient(subscriberId, namespace, topic string, partition int32, startTime time.Time) (messaging_pb.SeaweedMessaging_SubscribeClient, error) { - stream, err := messaging_pb.NewSeaweedMessagingClient(mc.grpcConnection).Subscribe(context.Background()) + stream, newBroker, err := mc.initSubscriberClient(subscriberId, namespace, topic, partition, startTime) if err != nil { - return nil, err + return client, err + } + if newBroker != nil { + + } + + return stream, nil + +} + +func setupSubscriberClient(grpcConnection *grpc.ClientConn, subscriberId string, namespace string, topic string, partition int32, startTime time.Time) (stream messaging_pb.SeaweedMessaging_SubscribeClient, err error) { + stream, err = messaging_pb.NewSeaweedMessagingClient(grpcConnection).Subscribe(context.Background()) + if err != nil { + return } // send init message @@ -53,20 +66,18 @@ func (mc *MessagingClient) setupSubscriberClient(subscriberId, namespace, topic }, }) if err != nil { - return nil, err + return } // process init response initResponse, err := stream.Recv() if err != nil { - return nil, err + return } if initResponse.Redirect != nil { // TODO follow redirection } - return stream, nil - } func (s *Subscriber) doSubscribe(partition int, processFn func(m *messaging_pb.Message)) error { diff --git a/weed/pb/filer.proto b/weed/pb/filer.proto index 6d890861d..3b3b78bbb 100644 --- a/weed/pb/filer.proto +++ b/weed/pb/filer.proto @@ -48,6 +48,8 @@ service SeaweedFiler { rpc SubscribeMetadata (SubscribeMetadataRequest) returns (stream SubscribeMetadataResponse) { } + rpc KeepConnected (stream KeepConnectedRequest) returns (stream KeepConnectedResponse) { + } } ////////////////////////////////////////////////// @@ -261,3 +263,10 @@ message LogEntry { int32 partition_key_hash = 2; bytes data = 3; } + +message KeepConnectedRequest { + string name = 1; + uint32 grpc_port = 2; +} +message KeepConnectedResponse { +} diff --git a/weed/pb/filer_pb/filer.pb.go b/weed/pb/filer_pb/filer.pb.go index 83f0eae0d..f1320273f 100644 --- a/weed/pb/filer_pb/filer.pb.go +++ b/weed/pb/filer_pb/filer.pb.go @@ -44,6 +44,8 @@ It has these top-level messages: SubscribeMetadataRequest SubscribeMetadataResponse LogEntry + KeepConnectedRequest + KeepConnectedResponse */ package filer_pb @@ -1228,6 +1230,38 @@ func (m *LogEntry) GetData() []byte { return nil } +type KeepConnectedRequest struct { + Name string `protobuf:"bytes,1,opt,name=name" json:"name,omitempty"` + GrpcPort uint32 `protobuf:"varint,2,opt,name=grpc_port,json=grpcPort" json:"grpc_port,omitempty"` +} + +func (m *KeepConnectedRequest) Reset() { *m = KeepConnectedRequest{} } +func (m *KeepConnectedRequest) String() string { return proto.CompactTextString(m) } +func (*KeepConnectedRequest) ProtoMessage() {} +func (*KeepConnectedRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{35} } + +func (m *KeepConnectedRequest) GetName() string { + if m != nil { + return m.Name + } + return "" +} + +func (m *KeepConnectedRequest) GetGrpcPort() uint32 { + if m != nil { + return m.GrpcPort + } + return 0 +} + +type KeepConnectedResponse struct { +} + +func (m *KeepConnectedResponse) Reset() { *m = KeepConnectedResponse{} } +func (m *KeepConnectedResponse) String() string { return proto.CompactTextString(m) } +func (*KeepConnectedResponse) ProtoMessage() {} +func (*KeepConnectedResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{36} } + func init() { proto.RegisterType((*LookupDirectoryEntryRequest)(nil), "filer_pb.LookupDirectoryEntryRequest") proto.RegisterType((*LookupDirectoryEntryResponse)(nil), "filer_pb.LookupDirectoryEntryResponse") @@ -1264,6 +1298,8 @@ func init() { proto.RegisterType((*SubscribeMetadataRequest)(nil), "filer_pb.SubscribeMetadataRequest") proto.RegisterType((*SubscribeMetadataResponse)(nil), "filer_pb.SubscribeMetadataResponse") proto.RegisterType((*LogEntry)(nil), "filer_pb.LogEntry") + proto.RegisterType((*KeepConnectedRequest)(nil), "filer_pb.KeepConnectedRequest") + proto.RegisterType((*KeepConnectedResponse)(nil), "filer_pb.KeepConnectedResponse") } // Reference imports to suppress errors if they are not otherwise used. @@ -1290,6 +1326,7 @@ type SeaweedFilerClient interface { Statistics(ctx context.Context, in *StatisticsRequest, opts ...grpc.CallOption) (*StatisticsResponse, error) GetFilerConfiguration(ctx context.Context, in *GetFilerConfigurationRequest, opts ...grpc.CallOption) (*GetFilerConfigurationResponse, error) SubscribeMetadata(ctx context.Context, in *SubscribeMetadataRequest, opts ...grpc.CallOption) (SeaweedFiler_SubscribeMetadataClient, error) + KeepConnected(ctx context.Context, opts ...grpc.CallOption) (SeaweedFiler_KeepConnectedClient, error) } type seaweedFilerClient struct { @@ -1463,6 +1500,37 @@ func (x *seaweedFilerSubscribeMetadataClient) Recv() (*SubscribeMetadataResponse return m, nil } +func (c *seaweedFilerClient) KeepConnected(ctx context.Context, opts ...grpc.CallOption) (SeaweedFiler_KeepConnectedClient, error) { + stream, err := grpc.NewClientStream(ctx, &_SeaweedFiler_serviceDesc.Streams[2], c.cc, "/filer_pb.SeaweedFiler/KeepConnected", opts...) + if err != nil { + return nil, err + } + x := &seaweedFilerKeepConnectedClient{stream} + return x, nil +} + +type SeaweedFiler_KeepConnectedClient interface { + Send(*KeepConnectedRequest) error + Recv() (*KeepConnectedResponse, error) + grpc.ClientStream +} + +type seaweedFilerKeepConnectedClient struct { + grpc.ClientStream +} + +func (x *seaweedFilerKeepConnectedClient) Send(m *KeepConnectedRequest) error { + return x.ClientStream.SendMsg(m) +} + +func (x *seaweedFilerKeepConnectedClient) Recv() (*KeepConnectedResponse, error) { + m := new(KeepConnectedResponse) + if err := x.ClientStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + // Server API for SeaweedFiler service type SeaweedFilerServer interface { @@ -1479,6 +1547,7 @@ type SeaweedFilerServer interface { Statistics(context.Context, *StatisticsRequest) (*StatisticsResponse, error) GetFilerConfiguration(context.Context, *GetFilerConfigurationRequest) (*GetFilerConfigurationResponse, error) SubscribeMetadata(*SubscribeMetadataRequest, SeaweedFiler_SubscribeMetadataServer) error + KeepConnected(SeaweedFiler_KeepConnectedServer) error } func RegisterSeaweedFilerServer(s *grpc.Server, srv SeaweedFilerServer) { @@ -1725,6 +1794,32 @@ func (x *seaweedFilerSubscribeMetadataServer) Send(m *SubscribeMetadataResponse) return x.ServerStream.SendMsg(m) } +func _SeaweedFiler_KeepConnected_Handler(srv interface{}, stream grpc.ServerStream) error { + return srv.(SeaweedFilerServer).KeepConnected(&seaweedFilerKeepConnectedServer{stream}) +} + +type SeaweedFiler_KeepConnectedServer interface { + Send(*KeepConnectedResponse) error + Recv() (*KeepConnectedRequest, error) + grpc.ServerStream +} + +type seaweedFilerKeepConnectedServer struct { + grpc.ServerStream +} + +func (x *seaweedFilerKeepConnectedServer) Send(m *KeepConnectedResponse) error { + return x.ServerStream.SendMsg(m) +} + +func (x *seaweedFilerKeepConnectedServer) Recv() (*KeepConnectedRequest, error) { + m := new(KeepConnectedRequest) + if err := x.ServerStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + var _SeaweedFiler_serviceDesc = grpc.ServiceDesc{ ServiceName: "filer_pb.SeaweedFiler", HandlerType: (*SeaweedFilerServer)(nil), @@ -1785,6 +1880,12 @@ var _SeaweedFiler_serviceDesc = grpc.ServiceDesc{ Handler: _SeaweedFiler_SubscribeMetadata_Handler, ServerStreams: true, }, + { + StreamName: "KeepConnected", + Handler: _SeaweedFiler_KeepConnected_Handler, + ServerStreams: true, + ClientStreams: true, + }, }, Metadata: "filer.proto", } @@ -1792,128 +1893,132 @@ var _SeaweedFiler_serviceDesc = grpc.ServiceDesc{ func init() { proto.RegisterFile("filer.proto", fileDescriptor0) } var fileDescriptor0 = []byte{ - // 1957 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xb4, 0x58, 0x5f, 0x6f, 0xdb, 0xc8, - 0x11, 0x0f, 0x25, 0x4b, 0x16, 0x47, 0x52, 0xce, 0x5e, 0xdb, 0x89, 0xac, 0xd8, 0x8e, 0x8f, 0x69, - 0xae, 0x29, 0x12, 0xb8, 0x81, 0x7b, 0x05, 0xee, 0x7a, 0xed, 0x43, 0xe2, 0x38, 0xd7, 0xf4, 0x12, - 0x5f, 0x40, 0x27, 0x45, 0x8b, 0x02, 0x65, 0x69, 0x72, 0x2d, 0x6d, 0x4d, 0x91, 0xec, 0xee, 0xd2, - 0x7f, 0xee, 0xe9, 0x3e, 0x47, 0x81, 0x7e, 0x8b, 0x3e, 0x16, 0x7d, 0x29, 0x0a, 0x14, 0xe8, 0xb7, - 0xe8, 0xf7, 0x28, 0x50, 0xec, 0x2c, 0x49, 0x2d, 0x45, 0xc9, 0xbe, 0xa0, 0xb8, 0xb7, 0xdd, 0x99, - 0xd9, 0xd9, 0xd9, 0xf9, 0xf3, 0x9b, 0x21, 0xa1, 0x7b, 0xca, 0x22, 0xca, 0xf7, 0x52, 0x9e, 0xc8, - 0x84, 0x74, 0x70, 0xe3, 0xa5, 0x27, 0xce, 0xd7, 0x70, 0xef, 0x75, 0x92, 0x9c, 0x65, 0xe9, 0x0b, - 0xc6, 0x69, 0x20, 0x13, 0x7e, 0x75, 0x18, 0x4b, 0x7e, 0xe5, 0xd2, 0x3f, 0x65, 0x54, 0x48, 0xb2, - 0x05, 0x76, 0x58, 0x30, 0x06, 0xd6, 0xae, 0xf5, 0xc8, 0x76, 0xa7, 0x04, 0x42, 0x60, 0x29, 0xf6, - 0x27, 0x74, 0xd0, 0x40, 0x06, 0xae, 0x9d, 0x43, 0xd8, 0x9a, 0xaf, 0x50, 0xa4, 0x49, 0x2c, 0x28, - 0x79, 0x08, 0x2d, 0xaa, 0x08, 0xa8, 0xad, 0xbb, 0xff, 0xd1, 0x5e, 0x61, 0xca, 0x9e, 0x96, 0xd3, - 0x5c, 0xe7, 0xef, 0x16, 0x90, 0xd7, 0x4c, 0x48, 0x45, 0x64, 0x54, 0x7c, 0x37, 0x7b, 0xee, 0x40, - 0x3b, 0xe5, 0xf4, 0x94, 0x5d, 0xe6, 0x16, 0xe5, 0x3b, 0xf2, 0x04, 0x56, 0x85, 0xf4, 0xb9, 0x7c, - 0xc9, 0x93, 0xc9, 0x4b, 0x16, 0xd1, 0x23, 0x65, 0x74, 0x13, 0x45, 0xea, 0x0c, 0xb2, 0x07, 0x84, - 0xc5, 0x41, 0x94, 0x09, 0x76, 0x4e, 0x8f, 0x0b, 0xee, 0x60, 0x69, 0xd7, 0x7a, 0xd4, 0x71, 0xe7, - 0x70, 0xc8, 0x3a, 0xb4, 0x22, 0x36, 0x61, 0x72, 0xd0, 0xda, 0xb5, 0x1e, 0xf5, 0x5d, 0xbd, 0x71, - 0x7e, 0x0e, 0x6b, 0x15, 0xfb, 0x3f, 0xec, 0xf9, 0x7f, 0x69, 0x40, 0x0b, 0x09, 0xa5, 0x8f, 0xad, - 0xa9, 0x8f, 0xc9, 0xc7, 0xd0, 0x63, 0xc2, 0x9b, 0x3a, 0xa2, 0x81, 0xb6, 0x75, 0x99, 0x28, 0x7d, - 0x4e, 0x1e, 0x43, 0x3b, 0x18, 0x67, 0xf1, 0x99, 0x18, 0x34, 0x77, 0x9b, 0x8f, 0xba, 0xfb, 0x6b, - 0xd3, 0x8b, 0xd4, 0x43, 0x0f, 0x14, 0xcf, 0xcd, 0x45, 0xc8, 0x67, 0x00, 0xbe, 0x94, 0x9c, 0x9d, - 0x64, 0x92, 0x0a, 0x7c, 0x69, 0x77, 0x7f, 0x60, 0x1c, 0xc8, 0x04, 0x7d, 0x56, 0xf2, 0x5d, 0x43, - 0x96, 0x7c, 0x0e, 0x1d, 0x7a, 0x29, 0x69, 0x1c, 0xd2, 0x70, 0xd0, 0xc2, 0x8b, 0xb6, 0x67, 0x5e, - 0xb4, 0x77, 0x98, 0xf3, 0xf5, 0xfb, 0x4a, 0xf1, 0xe1, 0x17, 0xd0, 0xaf, 0xb0, 0xc8, 0x0a, 0x34, - 0xcf, 0x68, 0x11, 0x55, 0xb5, 0x54, 0x9e, 0x3d, 0xf7, 0xa3, 0x4c, 0x27, 0x58, 0xcf, 0xd5, 0x9b, - 0x9f, 0x35, 0x3e, 0xb3, 0x9c, 0x17, 0x60, 0xbf, 0xcc, 0xa2, 0xa8, 0x3c, 0x18, 0x32, 0x5e, 0x1c, - 0x0c, 0x19, 0x9f, 0x7a, 0xb9, 0x71, 0xad, 0x97, 0xff, 0x66, 0xc1, 0xea, 0xe1, 0x39, 0x8d, 0xe5, - 0x51, 0x22, 0xd9, 0x29, 0x0b, 0x7c, 0xc9, 0x92, 0x98, 0x3c, 0x01, 0x3b, 0x89, 0x42, 0xef, 0xda, - 0x30, 0x75, 0x92, 0x28, 0xb7, 0xfa, 0x09, 0xd8, 0x31, 0xbd, 0xf0, 0xae, 0xbd, 0xae, 0x13, 0xd3, - 0x0b, 0x2d, 0xfd, 0x00, 0xfa, 0x21, 0x8d, 0xa8, 0xa4, 0x5e, 0x19, 0x1d, 0x15, 0xba, 0x9e, 0x26, - 0x1e, 0xe8, 0x70, 0x7c, 0x02, 0x1f, 0x29, 0x95, 0xa9, 0xcf, 0x69, 0x2c, 0xbd, 0xd4, 0x97, 0x63, - 0x8c, 0x89, 0xed, 0xf6, 0x63, 0x7a, 0xf1, 0x16, 0xa9, 0x6f, 0x7d, 0x39, 0x76, 0xfe, 0xda, 0x00, - 0xbb, 0x0c, 0x26, 0xb9, 0x0b, 0xcb, 0xea, 0x5a, 0x8f, 0x85, 0xb9, 0x27, 0xda, 0x6a, 0xfb, 0x2a, - 0x54, 0x55, 0x91, 0x9c, 0x9e, 0x0a, 0x2a, 0xd1, 0xbc, 0xa6, 0x9b, 0xef, 0x54, 0x66, 0x09, 0xf6, - 0x8d, 0x2e, 0x84, 0x25, 0x17, 0xd7, 0xca, 0xe3, 0x13, 0xc9, 0x26, 0x14, 0x2f, 0x6c, 0xba, 0x7a, - 0x43, 0xd6, 0xa0, 0x45, 0x3d, 0xe9, 0x8f, 0x30, 0xc3, 0x6d, 0x77, 0x89, 0xbe, 0xf3, 0x47, 0xe4, - 0x07, 0x70, 0x5b, 0x24, 0x19, 0x0f, 0xa8, 0x57, 0x5c, 0xdb, 0x46, 0x6e, 0x4f, 0x53, 0x5f, 0xea, - 0xcb, 0x1d, 0x68, 0x9e, 0xb2, 0x70, 0xb0, 0x8c, 0x8e, 0x59, 0xa9, 0x26, 0xe1, 0xab, 0xd0, 0x55, - 0x4c, 0xf2, 0x63, 0x80, 0x52, 0x53, 0x38, 0xe8, 0x2c, 0x10, 0xb5, 0x0b, 0xbd, 0x21, 0xd9, 0x06, - 0x08, 0x58, 0x3a, 0xa6, 0xdc, 0x53, 0x09, 0x63, 0x63, 0x72, 0xd8, 0x9a, 0xf2, 0x15, 0xbd, 0x52, - 0x6c, 0x26, 0xbc, 0xd1, 0x37, 0x2c, 0x4d, 0x69, 0x38, 0x00, 0xf4, 0xb0, 0xcd, 0xc4, 0x97, 0x9a, - 0xe0, 0xfc, 0x06, 0xda, 0xb9, 0x71, 0xf7, 0xc0, 0x3e, 0x4f, 0xa2, 0x6c, 0x52, 0x3a, 0xad, 0xef, - 0x76, 0x34, 0xe1, 0x55, 0x48, 0x36, 0x01, 0x51, 0x12, 0xaf, 0x68, 0xa0, 0x8b, 0xd0, 0xbf, 0xea, - 0x82, 0x3b, 0xd0, 0x0e, 0x92, 0xe4, 0x8c, 0x69, 0xdf, 0x2d, 0xbb, 0xf9, 0xce, 0xf9, 0xb6, 0x09, - 0xb7, 0xab, 0xc5, 0xa2, 0xae, 0x40, 0x2d, 0xe8, 0x69, 0x0b, 0xd5, 0xa0, 0xda, 0xe3, 0x8a, 0xb7, - 0x1b, 0xa6, 0xb7, 0x8b, 0x23, 0x93, 0x24, 0xd4, 0x17, 0xf4, 0xf5, 0x91, 0x37, 0x49, 0x48, 0x55, - 0xae, 0x67, 0x2c, 0xc4, 0xf0, 0xf4, 0x5d, 0xb5, 0x54, 0x94, 0x11, 0x0b, 0x73, 0xf0, 0x51, 0x4b, - 0x34, 0x8f, 0xa3, 0xde, 0xb6, 0x0e, 0xb8, 0xde, 0xa9, 0x80, 0x4f, 0x14, 0x75, 0x59, 0x47, 0x51, - 0xad, 0xc9, 0x2e, 0x74, 0x39, 0x4d, 0xa3, 0x3c, 0xf7, 0xd1, 0xf9, 0xb6, 0x6b, 0x92, 0xc8, 0x0e, - 0x40, 0x90, 0x44, 0x11, 0x0d, 0x50, 0xc0, 0x46, 0x01, 0x83, 0xa2, 0xf2, 0x4e, 0xca, 0xc8, 0x13, - 0x34, 0x40, 0x57, 0xb7, 0xdc, 0xb6, 0x94, 0xd1, 0x31, 0x0d, 0xd4, 0x3b, 0x32, 0x41, 0xb9, 0x87, - 0xf0, 0xd5, 0xc5, 0x73, 0x1d, 0x45, 0x40, 0x90, 0xdd, 0x06, 0x18, 0xf1, 0x24, 0x4b, 0x35, 0xb7, - 0xb7, 0xdb, 0x54, 0x48, 0x8e, 0x14, 0x64, 0x3f, 0x84, 0xdb, 0xe2, 0x6a, 0x12, 0xb1, 0xf8, 0xcc, - 0x93, 0x3e, 0x1f, 0x51, 0x39, 0xe8, 0xeb, 0x0a, 0xc8, 0xa9, 0xef, 0x90, 0xa8, 0xde, 0x3e, 0x09, - 0x7f, 0x3a, 0xb8, 0x8d, 0x19, 0xa0, 0x96, 0x4e, 0x0a, 0xe4, 0x80, 0x53, 0x5f, 0xd2, 0x0f, 0x68, - 0x63, 0xdf, 0x0d, 0x2d, 0xc8, 0x06, 0xb4, 0x13, 0x8f, 0x5e, 0x06, 0x51, 0x5e, 0xb4, 0xad, 0xe4, - 0xf0, 0x32, 0x88, 0x9c, 0xc7, 0xb0, 0x56, 0xb9, 0x31, 0x07, 0xfa, 0x75, 0x68, 0x51, 0xce, 0x93, - 0x02, 0x96, 0xf4, 0xc6, 0xf9, 0x2d, 0x90, 0xf7, 0x69, 0xf8, 0x7d, 0x98, 0xe7, 0x6c, 0xc0, 0x5a, - 0x45, 0xb5, 0xb6, 0xc3, 0xf9, 0xd6, 0x82, 0xf5, 0x67, 0x69, 0x4a, 0xe3, 0xf0, 0x5d, 0xf2, 0x01, - 0x97, 0x6e, 0x03, 0xa0, 0x5a, 0xcf, 0x68, 0xf0, 0x36, 0x52, 0x30, 0x3e, 0x1f, 0xd2, 0x5e, 0x9c, - 0xbb, 0xb0, 0x31, 0x63, 0x41, 0x6e, 0xdb, 0x3f, 0x2d, 0x20, 0x2f, 0x10, 0xf9, 0xfe, 0xbf, 0xa1, - 0x43, 0x61, 0x91, 0x6a, 0x88, 0x1a, 0x59, 0x43, 0x5f, 0xfa, 0x79, 0xbb, 0xee, 0x31, 0xa1, 0xf5, - 0xbf, 0xf0, 0xa5, 0x9f, 0xb7, 0x4d, 0x4e, 0x83, 0x8c, 0xab, 0x0e, 0x8e, 0x25, 0x83, 0x6d, 0xd3, - 0x2d, 0x48, 0xe4, 0x53, 0xb8, 0xc3, 0x46, 0x71, 0xc2, 0xe9, 0x54, 0xcc, 0xd3, 0x61, 0x6c, 0xa3, - 0xf0, 0xba, 0xe6, 0x96, 0x07, 0x0e, 0x31, 0xaa, 0x8f, 0x61, 0xad, 0xf2, 0x8c, 0x6b, 0x53, 0xe0, - 0xcf, 0x16, 0x0c, 0x9e, 0xc9, 0x64, 0xc2, 0x02, 0x97, 0x2a, 0xe3, 0x2b, 0x4f, 0x7f, 0x00, 0x7d, - 0xd5, 0x7b, 0x66, 0x9f, 0xdf, 0x4b, 0xa2, 0x70, 0xda, 0xdb, 0x37, 0x41, 0xb5, 0x1f, 0x33, 0x32, - 0xcb, 0x49, 0x14, 0x62, 0x5c, 0x1e, 0x80, 0xea, 0x11, 0xc6, 0x79, 0x3d, 0xe5, 0xf4, 0x62, 0x7a, - 0x51, 0x39, 0xaf, 0x84, 0xf0, 0xbc, 0x6e, 0x2c, 0xcb, 0x31, 0xbd, 0x50, 0xe7, 0x9d, 0x7b, 0xb0, - 0x39, 0xc7, 0xb6, 0x3c, 0x5c, 0xff, 0xb2, 0x60, 0xed, 0x99, 0x10, 0x6c, 0x14, 0xff, 0x1a, 0x41, - 0xb2, 0x30, 0x7a, 0x1d, 0x5a, 0x41, 0x92, 0xc5, 0x12, 0x8d, 0x6d, 0xb9, 0x7a, 0x33, 0x83, 0x1b, - 0x8d, 0x1a, 0x6e, 0xcc, 0x20, 0x4f, 0xb3, 0x8e, 0x3c, 0x06, 0xb2, 0x2c, 0x55, 0x90, 0xe5, 0x3e, - 0x74, 0x55, 0x90, 0xbd, 0x80, 0xc6, 0x92, 0xf2, 0xbc, 0x2b, 0x81, 0x22, 0x1d, 0x20, 0x45, 0x09, - 0x98, 0xdd, 0x53, 0x37, 0x26, 0x48, 0xa7, 0xad, 0xf3, 0x3f, 0xaa, 0x2a, 0x2a, 0x4f, 0xc9, 0x63, - 0xb6, 0xb0, 0x8b, 0x2a, 0xe0, 0xe5, 0x51, 0xfe, 0x0e, 0xb5, 0x54, 0x25, 0x92, 0x66, 0x27, 0x11, - 0x0b, 0x3c, 0xc5, 0xd0, 0xf6, 0xdb, 0x9a, 0xf2, 0x9e, 0x47, 0x53, 0xaf, 0x2c, 0x99, 0x5e, 0x21, - 0xb0, 0xe4, 0x67, 0x72, 0x5c, 0x74, 0x52, 0xb5, 0x9e, 0xf1, 0x54, 0xfb, 0x26, 0x4f, 0x2d, 0xd7, - 0x3d, 0x55, 0x66, 0x5a, 0xc7, 0xcc, 0xb4, 0x4f, 0x61, 0x4d, 0x8f, 0xe2, 0xd5, 0x70, 0x6d, 0x03, - 0x94, 0x5d, 0x4f, 0x0c, 0x2c, 0x0d, 0xbd, 0x45, 0xdb, 0x13, 0xce, 0x2f, 0xc0, 0x7e, 0x9d, 0x68, - 0xbd, 0x82, 0x3c, 0x05, 0x3b, 0x2a, 0x36, 0x28, 0xda, 0xdd, 0x27, 0xd3, 0x52, 0x2f, 0xe4, 0xdc, - 0xa9, 0x90, 0xf3, 0x05, 0x74, 0x0a, 0x72, 0xe1, 0x33, 0x6b, 0x91, 0xcf, 0x1a, 0x33, 0x3e, 0x73, - 0xfe, 0x61, 0xc1, 0x7a, 0xd5, 0xe4, 0x3c, 0x2c, 0xef, 0xa1, 0x5f, 0x5e, 0xe1, 0x4d, 0xfc, 0x34, - 0xb7, 0xe5, 0xa9, 0x69, 0x4b, 0xfd, 0x58, 0x69, 0xa0, 0x78, 0xe3, 0xa7, 0x3a, 0x97, 0x7b, 0x91, - 0x41, 0x1a, 0xbe, 0x83, 0xd5, 0x9a, 0xc8, 0x9c, 0x39, 0xf4, 0x47, 0xe6, 0x1c, 0x5a, 0x01, 0xbb, - 0xf2, 0xb4, 0x39, 0x9c, 0x7e, 0x0e, 0x77, 0x35, 0x1c, 0x1c, 0x94, 0x31, 0x2c, 0x7c, 0x5f, 0x0d, - 0xb5, 0x35, 0x1b, 0x6a, 0x67, 0x08, 0x83, 0xfa, 0xd1, 0xbc, 0xfc, 0x46, 0xb0, 0x7a, 0x2c, 0x7d, - 0xc9, 0x84, 0x64, 0x41, 0xf9, 0x41, 0x34, 0x93, 0x1b, 0xd6, 0x4d, 0xfd, 0xbb, 0x5e, 0x87, 0x2b, - 0xd0, 0x94, 0xb2, 0xc8, 0x5f, 0xb5, 0x54, 0x51, 0x20, 0xe6, 0x4d, 0x79, 0x0c, 0xbe, 0x87, 0xab, - 0x54, 0x3e, 0xc8, 0x44, 0xfa, 0x91, 0x9e, 0x8f, 0x96, 0x70, 0x3e, 0xb2, 0x91, 0x82, 0x03, 0x92, - 0x1e, 0x21, 0x42, 0xcd, 0x6d, 0xe9, 0xe9, 0x49, 0x11, 0x90, 0xb9, 0x0d, 0x80, 0xa5, 0xaa, 0xab, - 0xac, 0xad, 0xcf, 0x2a, 0xca, 0x81, 0x22, 0x38, 0x3b, 0xb0, 0xf5, 0x25, 0x95, 0xaa, 0x1b, 0xf1, - 0x83, 0x24, 0x3e, 0x65, 0xa3, 0x8c, 0xfb, 0x46, 0x28, 0x9c, 0x7f, 0x5b, 0xb0, 0xbd, 0x40, 0x20, - 0x7f, 0xf0, 0x00, 0x96, 0x27, 0xbe, 0x90, 0x94, 0x17, 0x55, 0x52, 0x6c, 0x67, 0x5d, 0xd1, 0xb8, - 0xc9, 0x15, 0xcd, 0x9a, 0x2b, 0x36, 0xa0, 0x3d, 0xf1, 0x2f, 0xbd, 0xc9, 0x49, 0x3e, 0xca, 0xb5, - 0x26, 0xfe, 0xe5, 0x9b, 0x13, 0x44, 0x36, 0xc6, 0xbd, 0x93, 0x2c, 0x38, 0xa3, 0x52, 0x94, 0xc8, - 0xc6, 0xf8, 0x73, 0x4d, 0xc1, 0xd9, 0x0e, 0x07, 0x5d, 0x84, 0x81, 0x8e, 0x9b, 0xef, 0x9c, 0x0b, - 0x18, 0x1c, 0x67, 0x27, 0x22, 0xe0, 0xec, 0x84, 0xbe, 0xa1, 0xd2, 0x57, 0x60, 0x58, 0xe4, 0xc8, - 0x7d, 0xe8, 0x06, 0x11, 0x53, 0x68, 0x68, 0x7c, 0x49, 0x82, 0x26, 0x61, 0xd7, 0x40, 0xb8, 0x94, - 0x63, 0xaf, 0xf2, 0xf1, 0x0c, 0x8a, 0xf4, 0x56, 0x7f, 0x40, 0x6f, 0x42, 0x47, 0xb0, 0x38, 0xa0, - 0x5e, 0xac, 0xbf, 0x58, 0x9a, 0xee, 0x32, 0xee, 0x8f, 0x84, 0x6a, 0x67, 0x9b, 0x73, 0x6e, 0xce, - 0x5d, 0x78, 0x7d, 0x2b, 0xff, 0x15, 0x10, 0x7a, 0x8e, 0x76, 0x19, 0xdf, 0x5f, 0x79, 0x91, 0xdd, - 0x33, 0xc6, 0x9c, 0xd9, 0x4f, 0x34, 0x77, 0x95, 0xd6, 0xbe, 0xda, 0xd6, 0xa0, 0x25, 0xc5, 0xd4, - 0xbe, 0x25, 0x29, 0x8e, 0x84, 0xe3, 0x2b, 0x30, 0x1a, 0xe9, 0xb2, 0x2e, 0x05, 0xac, 0xa9, 0x00, - 0x79, 0x02, 0x24, 0xf5, 0xb9, 0x64, 0x4a, 0x85, 0x9a, 0xf4, 0xbd, 0xb1, 0x2f, 0xc6, 0x68, 0x41, - 0xcb, 0x5d, 0x29, 0x39, 0x5f, 0xd1, 0xab, 0x5f, 0xfa, 0x62, 0xac, 0xc0, 0x1b, 0x87, 0x8b, 0x26, - 0xce, 0x9b, 0xb8, 0xde, 0xff, 0x6f, 0x07, 0x7a, 0xc7, 0xd4, 0xbf, 0xa0, 0x34, 0xc4, 0x54, 0x22, - 0xa3, 0x02, 0xc2, 0xaa, 0x3f, 0x40, 0xc8, 0xc3, 0x59, 0xac, 0x9a, 0xfb, 0xc7, 0x65, 0xf8, 0xc9, - 0x4d, 0x62, 0x39, 0x1a, 0xdc, 0x22, 0x47, 0xd0, 0x35, 0xfe, 0x30, 0x90, 0x2d, 0xe3, 0x60, 0xed, - 0xc7, 0xc9, 0x70, 0x7b, 0x01, 0xb7, 0xd0, 0xf6, 0xd4, 0x22, 0xaf, 0xa1, 0x6b, 0x0c, 0xb2, 0xa6, - 0xbe, 0xfa, 0x44, 0x6d, 0xea, 0x9b, 0x33, 0xfd, 0x3a, 0xb7, 0x94, 0x36, 0x63, 0x1c, 0x35, 0xb5, - 0xd5, 0x07, 0x60, 0x53, 0xdb, 0xbc, 0x19, 0xf6, 0x16, 0x71, 0xa1, 0x5f, 0x19, 0x21, 0xc9, 0xce, - 0xf4, 0xc4, 0xbc, 0xe9, 0x76, 0x78, 0x7f, 0x21, 0xdf, 0xb4, 0xd0, 0x98, 0xda, 0x4c, 0x0b, 0xeb, - 0x33, 0xa9, 0x69, 0xe1, 0x9c, 0x51, 0xcf, 0xb9, 0x45, 0x7e, 0x0f, 0xab, 0xb5, 0xc9, 0x89, 0x38, - 0x86, 0x15, 0x0b, 0x46, 0xbe, 0xe1, 0x83, 0x6b, 0x65, 0x4a, 0xfd, 0x5f, 0x43, 0xcf, 0x1c, 0x58, - 0x88, 0x61, 0xd0, 0x9c, 0x99, 0x6c, 0xb8, 0xb3, 0x88, 0x6d, 0x2a, 0x34, 0x7b, 0xa6, 0xa9, 0x70, - 0xce, 0xd4, 0x60, 0x2a, 0x9c, 0xd7, 0x6a, 0x9d, 0x5b, 0xe4, 0x77, 0xb0, 0x32, 0xdb, 0xbb, 0xc8, - 0xc7, 0xb3, 0x6e, 0xab, 0xb5, 0xc4, 0xa1, 0x73, 0x9d, 0x48, 0xa9, 0xfc, 0x15, 0xc0, 0xb4, 0x25, - 0x11, 0x03, 0x1c, 0x6a, 0x2d, 0x71, 0xb8, 0x35, 0x9f, 0x59, 0xaa, 0xfa, 0x23, 0x6c, 0xcc, 0xc5, - 0x7d, 0x62, 0x94, 0xde, 0x75, 0x9d, 0x63, 0xf8, 0xc3, 0x1b, 0xe5, 0xca, 0xbb, 0xfe, 0x00, 0xab, - 0x35, 0x70, 0x34, 0xb3, 0x62, 0x11, 0x66, 0x9b, 0x59, 0xb1, 0x10, 0x5d, 0x55, 0xd5, 0x3e, 0xdf, - 0x81, 0x15, 0xa1, 0xe1, 0xe7, 0x54, 0xec, 0x69, 0x4c, 0x7f, 0x0e, 0x68, 0xd3, 0x5b, 0x9e, 0xc8, - 0xe4, 0xa4, 0x8d, 0x7f, 0x7c, 0x7f, 0xf2, 0xbf, 0x00, 0x00, 0x00, 0xff, 0xff, 0x39, 0x78, 0xfe, - 0x97, 0x00, 0x16, 0x00, 0x00, + // 2020 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xb4, 0x58, 0xcd, 0x6e, 0xdc, 0xc8, + 0x11, 0x16, 0x67, 0x34, 0xa3, 0x61, 0xcd, 0x8c, 0x57, 0x6a, 0x49, 0xf6, 0x68, 0xf4, 0x63, 0x2d, + 0x1d, 0x6f, 0x14, 0xd8, 0x50, 0x0c, 0x65, 0x03, 0xec, 0x66, 0x93, 0x83, 0x2d, 0xcb, 0x8e, 0x63, + 0x5b, 0x2b, 0x50, 0xf6, 0x22, 0x41, 0x80, 0x30, 0x14, 0xd9, 0x1a, 0x75, 0xc4, 0x21, 0x99, 0xee, + 0xa6, 0x7e, 0xf6, 0xb4, 0xcf, 0x11, 0x20, 0x6f, 0x91, 0x63, 0x90, 0x4b, 0x10, 0x20, 0x40, 0xce, + 0x79, 0x81, 0x3c, 0x49, 0xd0, 0xd5, 0x24, 0xa7, 0x39, 0x3f, 0xd2, 0x1a, 0x0b, 0xdf, 0xba, 0xab, + 0xaa, 0xab, 0xab, 0xeb, 0xe7, 0xab, 0x22, 0xa1, 0x7d, 0xca, 0x22, 0xca, 0x77, 0x53, 0x9e, 0xc8, + 0x84, 0xb4, 0x70, 0xe3, 0xa5, 0x27, 0xce, 0xd7, 0xb0, 0xfe, 0x26, 0x49, 0xce, 0xb3, 0xf4, 0x39, + 0xe3, 0x34, 0x90, 0x09, 0xbf, 0x3e, 0x88, 0x25, 0xbf, 0x76, 0xe9, 0x9f, 0x33, 0x2a, 0x24, 0xd9, + 0x00, 0x3b, 0x2c, 0x18, 0x3d, 0x6b, 0xdb, 0xda, 0xb1, 0xdd, 0x11, 0x81, 0x10, 0x98, 0x8f, 0xfd, + 0x21, 0xed, 0xd5, 0x90, 0x81, 0x6b, 0xe7, 0x00, 0x36, 0xa6, 0x2b, 0x14, 0x69, 0x12, 0x0b, 0x4a, + 0x1e, 0x42, 0x83, 0x2a, 0x02, 0x6a, 0x6b, 0xef, 0x7d, 0xb2, 0x5b, 0x98, 0xb2, 0xab, 0xe5, 0x34, + 0xd7, 0xf9, 0x87, 0x05, 0xe4, 0x0d, 0x13, 0x52, 0x11, 0x19, 0x15, 0xdf, 0xcf, 0x9e, 0xbb, 0xd0, + 0x4c, 0x39, 0x3d, 0x65, 0x57, 0xb9, 0x45, 0xf9, 0x8e, 0x3c, 0x86, 0x25, 0x21, 0x7d, 0x2e, 0x5f, + 0xf0, 0x64, 0xf8, 0x82, 0x45, 0xf4, 0x50, 0x19, 0x5d, 0x47, 0x91, 0x49, 0x06, 0xd9, 0x05, 0xc2, + 0xe2, 0x20, 0xca, 0x04, 0xbb, 0xa0, 0xc7, 0x05, 0xb7, 0x37, 0xbf, 0x6d, 0xed, 0xb4, 0xdc, 0x29, + 0x1c, 0xb2, 0x02, 0x8d, 0x88, 0x0d, 0x99, 0xec, 0x35, 0xb6, 0xad, 0x9d, 0xae, 0xab, 0x37, 0xce, + 0x2f, 0x61, 0xb9, 0x62, 0xff, 0x87, 0x3d, 0xff, 0xaf, 0x35, 0x68, 0x20, 0xa1, 0xf4, 0xb1, 0x35, + 0xf2, 0x31, 0xf9, 0x14, 0x3a, 0x4c, 0x78, 0x23, 0x47, 0xd4, 0xd0, 0xb6, 0x36, 0x13, 0xa5, 0xcf, + 0xc9, 0x23, 0x68, 0x06, 0x67, 0x59, 0x7c, 0x2e, 0x7a, 0xf5, 0xed, 0xfa, 0x4e, 0x7b, 0x6f, 0x79, + 0x74, 0x91, 0x7a, 0xe8, 0xbe, 0xe2, 0xb9, 0xb9, 0x08, 0xf9, 0x02, 0xc0, 0x97, 0x92, 0xb3, 0x93, + 0x4c, 0x52, 0x81, 0x2f, 0x6d, 0xef, 0xf5, 0x8c, 0x03, 0x99, 0xa0, 0x4f, 0x4b, 0xbe, 0x6b, 0xc8, + 0x92, 0x2f, 0xa1, 0x45, 0xaf, 0x24, 0x8d, 0x43, 0x1a, 0xf6, 0x1a, 0x78, 0xd1, 0xe6, 0xd8, 0x8b, + 0x76, 0x0f, 0x72, 0xbe, 0x7e, 0x5f, 0x29, 0xde, 0xff, 0x0a, 0xba, 0x15, 0x16, 0x59, 0x84, 0xfa, + 0x39, 0x2d, 0xa2, 0xaa, 0x96, 0xca, 0xb3, 0x17, 0x7e, 0x94, 0xe9, 0x04, 0xeb, 0xb8, 0x7a, 0xf3, + 0x8b, 0xda, 0x17, 0x96, 0xf3, 0x1c, 0xec, 0x17, 0x59, 0x14, 0x95, 0x07, 0x43, 0xc6, 0x8b, 0x83, + 0x21, 0xe3, 0x23, 0x2f, 0xd7, 0x6e, 0xf4, 0xf2, 0xdf, 0x2d, 0x58, 0x3a, 0xb8, 0xa0, 0xb1, 0x3c, + 0x4c, 0x24, 0x3b, 0x65, 0x81, 0x2f, 0x59, 0x12, 0x93, 0xc7, 0x60, 0x27, 0x51, 0xe8, 0xdd, 0x18, + 0xa6, 0x56, 0x12, 0xe5, 0x56, 0x3f, 0x06, 0x3b, 0xa6, 0x97, 0xde, 0x8d, 0xd7, 0xb5, 0x62, 0x7a, + 0xa9, 0xa5, 0x1f, 0x40, 0x37, 0xa4, 0x11, 0x95, 0xd4, 0x2b, 0xa3, 0xa3, 0x42, 0xd7, 0xd1, 0xc4, + 0x7d, 0x1d, 0x8e, 0xcf, 0xe0, 0x13, 0xa5, 0x32, 0xf5, 0x39, 0x8d, 0xa5, 0x97, 0xfa, 0xf2, 0x0c, + 0x63, 0x62, 0xbb, 0xdd, 0x98, 0x5e, 0x1e, 0x21, 0xf5, 0xc8, 0x97, 0x67, 0xce, 0xdf, 0x6a, 0x60, + 0x97, 0xc1, 0x24, 0xf7, 0x60, 0x41, 0x5d, 0xeb, 0xb1, 0x30, 0xf7, 0x44, 0x53, 0x6d, 0x5f, 0x85, + 0xaa, 0x2a, 0x92, 0xd3, 0x53, 0x41, 0x25, 0x9a, 0x57, 0x77, 0xf3, 0x9d, 0xca, 0x2c, 0xc1, 0xbe, + 0xd5, 0x85, 0x30, 0xef, 0xe2, 0x5a, 0x79, 0x7c, 0x28, 0xd9, 0x90, 0xe2, 0x85, 0x75, 0x57, 0x6f, + 0xc8, 0x32, 0x34, 0xa8, 0x27, 0xfd, 0x01, 0x66, 0xb8, 0xed, 0xce, 0xd3, 0x77, 0xfe, 0x80, 0xfc, + 0x08, 0xee, 0x88, 0x24, 0xe3, 0x01, 0xf5, 0x8a, 0x6b, 0x9b, 0xc8, 0xed, 0x68, 0xea, 0x0b, 0x7d, + 0xb9, 0x03, 0xf5, 0x53, 0x16, 0xf6, 0x16, 0xd0, 0x31, 0x8b, 0xd5, 0x24, 0x7c, 0x15, 0xba, 0x8a, + 0x49, 0x7e, 0x0a, 0x50, 0x6a, 0x0a, 0x7b, 0xad, 0x19, 0xa2, 0x76, 0xa1, 0x37, 0x24, 0x9b, 0x00, + 0x01, 0x4b, 0xcf, 0x28, 0xf7, 0x54, 0xc2, 0xd8, 0x98, 0x1c, 0xb6, 0xa6, 0xbc, 0xa6, 0xd7, 0x8a, + 0xcd, 0x84, 0x37, 0xf8, 0x96, 0xa5, 0x29, 0x0d, 0x7b, 0x80, 0x1e, 0xb6, 0x99, 0x78, 0xa9, 0x09, + 0xce, 0x6f, 0xa1, 0x99, 0x1b, 0xb7, 0x0e, 0xf6, 0x45, 0x12, 0x65, 0xc3, 0xd2, 0x69, 0x5d, 0xb7, + 0xa5, 0x09, 0xaf, 0x42, 0xb2, 0x06, 0x88, 0x92, 0x78, 0x45, 0x0d, 0x5d, 0x84, 0xfe, 0x55, 0x17, + 0xdc, 0x85, 0x66, 0x90, 0x24, 0xe7, 0x4c, 0xfb, 0x6e, 0xc1, 0xcd, 0x77, 0xce, 0x77, 0x75, 0xb8, + 0x53, 0x2d, 0x16, 0x75, 0x05, 0x6a, 0x41, 0x4f, 0x5b, 0xa8, 0x06, 0xd5, 0x1e, 0x57, 0xbc, 0x5d, + 0x33, 0xbd, 0x5d, 0x1c, 0x19, 0x26, 0xa1, 0xbe, 0xa0, 0xab, 0x8f, 0xbc, 0x4d, 0x42, 0xaa, 0x72, + 0x3d, 0x63, 0x21, 0x86, 0xa7, 0xeb, 0xaa, 0xa5, 0xa2, 0x0c, 0x58, 0x98, 0x83, 0x8f, 0x5a, 0xa2, + 0x79, 0x1c, 0xf5, 0x36, 0x75, 0xc0, 0xf5, 0x4e, 0x05, 0x7c, 0xa8, 0xa8, 0x0b, 0x3a, 0x8a, 0x6a, + 0x4d, 0xb6, 0xa1, 0xcd, 0x69, 0x1a, 0xe5, 0xb9, 0x8f, 0xce, 0xb7, 0x5d, 0x93, 0x44, 0xb6, 0x00, + 0x82, 0x24, 0x8a, 0x68, 0x80, 0x02, 0x36, 0x0a, 0x18, 0x14, 0x95, 0x77, 0x52, 0x46, 0x9e, 0xa0, + 0x01, 0xba, 0xba, 0xe1, 0x36, 0xa5, 0x8c, 0x8e, 0x69, 0xa0, 0xde, 0x91, 0x09, 0xca, 0x3d, 0x84, + 0xaf, 0x36, 0x9e, 0x6b, 0x29, 0x02, 0x82, 0xec, 0x26, 0xc0, 0x80, 0x27, 0x59, 0xaa, 0xb9, 0x9d, + 0xed, 0xba, 0x42, 0x72, 0xa4, 0x20, 0xfb, 0x21, 0xdc, 0x11, 0xd7, 0xc3, 0x88, 0xc5, 0xe7, 0x9e, + 0xf4, 0xf9, 0x80, 0xca, 0x5e, 0x57, 0x57, 0x40, 0x4e, 0x7d, 0x87, 0x44, 0xf5, 0xf6, 0x61, 0xf8, + 0xf3, 0xde, 0x1d, 0xcc, 0x00, 0xb5, 0x74, 0x52, 0x20, 0xfb, 0x9c, 0xfa, 0x92, 0x7e, 0x40, 0x1b, + 0xfb, 0x7e, 0x68, 0x41, 0x56, 0xa1, 0x99, 0x78, 0xf4, 0x2a, 0x88, 0xf2, 0xa2, 0x6d, 0x24, 0x07, + 0x57, 0x41, 0xe4, 0x3c, 0x82, 0xe5, 0xca, 0x8d, 0x39, 0xd0, 0xaf, 0x40, 0x83, 0x72, 0x9e, 0x14, + 0xb0, 0xa4, 0x37, 0xce, 0xef, 0x80, 0xbc, 0x4f, 0xc3, 0x8f, 0x61, 0x9e, 0xb3, 0x0a, 0xcb, 0x15, + 0xd5, 0xda, 0x0e, 0xe7, 0x3b, 0x0b, 0x56, 0x9e, 0xa6, 0x29, 0x8d, 0xc3, 0x77, 0xc9, 0x07, 0x5c, + 0xba, 0x09, 0x80, 0x6a, 0x3d, 0xa3, 0xc1, 0xdb, 0x48, 0xc1, 0xf8, 0x7c, 0x48, 0x7b, 0x71, 0xee, + 0xc1, 0xea, 0x98, 0x05, 0xb9, 0x6d, 0xff, 0xb2, 0x80, 0x3c, 0x47, 0xe4, 0xfb, 0x61, 0x43, 0x87, + 0xc2, 0x22, 0xd5, 0x10, 0x35, 0xb2, 0x86, 0xbe, 0xf4, 0xf3, 0x76, 0xdd, 0x61, 0x42, 0xeb, 0x7f, + 0xee, 0x4b, 0x3f, 0x6f, 0x9b, 0x9c, 0x06, 0x19, 0x57, 0x1d, 0x1c, 0x4b, 0x06, 0xdb, 0xa6, 0x5b, + 0x90, 0xc8, 0xe7, 0x70, 0x97, 0x0d, 0xe2, 0x84, 0xd3, 0x91, 0x98, 0xa7, 0xc3, 0xd8, 0x44, 0xe1, + 0x15, 0xcd, 0x2d, 0x0f, 0x1c, 0x60, 0x54, 0x1f, 0xc1, 0x72, 0xe5, 0x19, 0x37, 0xa6, 0xc0, 0x5f, + 0x2c, 0xe8, 0x3d, 0x95, 0xc9, 0x90, 0x05, 0x2e, 0x55, 0xc6, 0x57, 0x9e, 0xfe, 0x00, 0xba, 0xaa, + 0xf7, 0x8c, 0x3f, 0xbf, 0x93, 0x44, 0xe1, 0xa8, 0xb7, 0xaf, 0x81, 0x6a, 0x3f, 0x66, 0x64, 0x16, + 0x92, 0x28, 0xc4, 0xb8, 0x3c, 0x00, 0xd5, 0x23, 0x8c, 0xf3, 0x7a, 0xca, 0xe9, 0xc4, 0xf4, 0xb2, + 0x72, 0x5e, 0x09, 0xe1, 0x79, 0xdd, 0x58, 0x16, 0x62, 0x7a, 0xa9, 0xce, 0x3b, 0xeb, 0xb0, 0x36, + 0xc5, 0xb6, 0x3c, 0x5c, 0xff, 0xb6, 0x60, 0xf9, 0xa9, 0x10, 0x6c, 0x10, 0x7f, 0x83, 0x20, 0x59, + 0x18, 0xbd, 0x02, 0x8d, 0x20, 0xc9, 0x62, 0x89, 0xc6, 0x36, 0x5c, 0xbd, 0x19, 0xc3, 0x8d, 0xda, + 0x04, 0x6e, 0x8c, 0x21, 0x4f, 0x7d, 0x12, 0x79, 0x0c, 0x64, 0x99, 0xaf, 0x20, 0xcb, 0x7d, 0x68, + 0xab, 0x20, 0x7b, 0x01, 0x8d, 0x25, 0xe5, 0x79, 0x57, 0x02, 0x45, 0xda, 0x47, 0x8a, 0x12, 0x30, + 0xbb, 0xa7, 0x6e, 0x4c, 0x90, 0x8e, 0x5a, 0xe7, 0xff, 0x54, 0x55, 0x54, 0x9e, 0x92, 0xc7, 0x6c, + 0x66, 0x17, 0x55, 0xc0, 0xcb, 0xa3, 0xfc, 0x1d, 0x6a, 0xa9, 0x4a, 0x24, 0xcd, 0x4e, 0x22, 0x16, + 0x78, 0x8a, 0xa1, 0xed, 0xb7, 0x35, 0xe5, 0x3d, 0x8f, 0x46, 0x5e, 0x99, 0x37, 0xbd, 0x42, 0x60, + 0xde, 0xcf, 0xe4, 0x59, 0xd1, 0x49, 0xd5, 0x7a, 0xcc, 0x53, 0xcd, 0xdb, 0x3c, 0xb5, 0x30, 0xe9, + 0xa9, 0x32, 0xd3, 0x5a, 0x66, 0xa6, 0x7d, 0x0e, 0xcb, 0x7a, 0x14, 0xaf, 0x86, 0x6b, 0x13, 0xa0, + 0xec, 0x7a, 0xa2, 0x67, 0x69, 0xe8, 0x2d, 0xda, 0x9e, 0x70, 0x7e, 0x05, 0xf6, 0x9b, 0x44, 0xeb, + 0x15, 0xe4, 0x09, 0xd8, 0x51, 0xb1, 0x41, 0xd1, 0xf6, 0x1e, 0x19, 0x95, 0x7a, 0x21, 0xe7, 0x8e, + 0x84, 0x9c, 0xaf, 0xa0, 0x55, 0x90, 0x0b, 0x9f, 0x59, 0xb3, 0x7c, 0x56, 0x1b, 0xf3, 0x99, 0xf3, + 0x4f, 0x0b, 0x56, 0xaa, 0x26, 0xe7, 0x61, 0x79, 0x0f, 0xdd, 0xf2, 0x0a, 0x6f, 0xe8, 0xa7, 0xb9, + 0x2d, 0x4f, 0x4c, 0x5b, 0x26, 0x8f, 0x95, 0x06, 0x8a, 0xb7, 0x7e, 0xaa, 0x73, 0xb9, 0x13, 0x19, + 0xa4, 0xfe, 0x3b, 0x58, 0x9a, 0x10, 0x99, 0x32, 0x87, 0xfe, 0xc4, 0x9c, 0x43, 0x2b, 0x60, 0x57, + 0x9e, 0x36, 0x87, 0xd3, 0x2f, 0xe1, 0x9e, 0x86, 0x83, 0xfd, 0x32, 0x86, 0x85, 0xef, 0xab, 0xa1, + 0xb6, 0xc6, 0x43, 0xed, 0xf4, 0xa1, 0x37, 0x79, 0x34, 0x2f, 0xbf, 0x01, 0x2c, 0x1d, 0x4b, 0x5f, + 0x32, 0x21, 0x59, 0x50, 0x7e, 0x10, 0x8d, 0xe5, 0x86, 0x75, 0x5b, 0xff, 0x9e, 0xac, 0xc3, 0x45, + 0xa8, 0x4b, 0x59, 0xe4, 0xaf, 0x5a, 0xaa, 0x28, 0x10, 0xf3, 0xa6, 0x3c, 0x06, 0x1f, 0xe1, 0x2a, + 0x95, 0x0f, 0x32, 0x91, 0x7e, 0xa4, 0xe7, 0xa3, 0x79, 0x9c, 0x8f, 0x6c, 0xa4, 0xe0, 0x80, 0xa4, + 0x47, 0x88, 0x50, 0x73, 0x1b, 0x7a, 0x7a, 0x52, 0x04, 0x64, 0x6e, 0x02, 0x60, 0xa9, 0xea, 0x2a, + 0x6b, 0xea, 0xb3, 0x8a, 0xb2, 0xaf, 0x08, 0xce, 0x16, 0x6c, 0xbc, 0xa4, 0x52, 0x75, 0x23, 0xbe, + 0x9f, 0xc4, 0xa7, 0x6c, 0x90, 0x71, 0xdf, 0x08, 0x85, 0xf3, 0x1f, 0x0b, 0x36, 0x67, 0x08, 0xe4, + 0x0f, 0xee, 0xc1, 0xc2, 0xd0, 0x17, 0x92, 0xf2, 0xa2, 0x4a, 0x8a, 0xed, 0xb8, 0x2b, 0x6a, 0xb7, + 0xb9, 0xa2, 0x3e, 0xe1, 0x8a, 0x55, 0x68, 0x0e, 0xfd, 0x2b, 0x6f, 0x78, 0x92, 0x8f, 0x72, 0x8d, + 0xa1, 0x7f, 0xf5, 0xf6, 0x04, 0x91, 0x8d, 0x71, 0xef, 0x24, 0x0b, 0xce, 0xa9, 0x14, 0x25, 0xb2, + 0x31, 0xfe, 0x4c, 0x53, 0x70, 0xb6, 0xc3, 0x41, 0x17, 0x61, 0xa0, 0xe5, 0xe6, 0x3b, 0xe7, 0x12, + 0x7a, 0xc7, 0xd9, 0x89, 0x08, 0x38, 0x3b, 0xa1, 0x6f, 0xa9, 0xf4, 0x15, 0x18, 0x16, 0x39, 0x72, + 0x1f, 0xda, 0x41, 0xc4, 0x14, 0x1a, 0x1a, 0x5f, 0x92, 0xa0, 0x49, 0xd8, 0x35, 0x10, 0x2e, 0xe5, + 0x99, 0x57, 0xf9, 0x78, 0x06, 0x45, 0x3a, 0xd2, 0x1f, 0xd0, 0x6b, 0xd0, 0x12, 0x2c, 0x0e, 0xa8, + 0x17, 0xeb, 0x2f, 0x96, 0xba, 0xbb, 0x80, 0xfb, 0x43, 0xa1, 0xda, 0xd9, 0xda, 0x94, 0x9b, 0x73, + 0x17, 0xde, 0xdc, 0xca, 0x7f, 0x03, 0x84, 0x5e, 0xa0, 0x5d, 0xc6, 0xf7, 0x57, 0x5e, 0x64, 0xeb, + 0xc6, 0x98, 0x33, 0xfe, 0x89, 0xe6, 0x2e, 0xd1, 0x89, 0xaf, 0xb6, 0x65, 0x68, 0x48, 0x31, 0xb2, + 0x6f, 0x5e, 0x8a, 0x43, 0xe1, 0xf8, 0x0a, 0x8c, 0x06, 0xba, 0xac, 0x4b, 0x01, 0x6b, 0x24, 0x40, + 0x1e, 0x03, 0x49, 0x7d, 0x2e, 0x99, 0x52, 0xa1, 0x26, 0x7d, 0xef, 0xcc, 0x17, 0x67, 0x68, 0x41, + 0xc3, 0x5d, 0x2c, 0x39, 0xaf, 0xe9, 0xf5, 0xaf, 0x7d, 0x71, 0xa6, 0xc0, 0x1b, 0x87, 0x8b, 0x3a, + 0xce, 0x9b, 0xb8, 0x76, 0x5e, 0xc2, 0xca, 0x6b, 0x4a, 0xd3, 0xfd, 0x24, 0x8e, 0x69, 0x20, 0x69, + 0x58, 0x38, 0x7d, 0xda, 0x77, 0xfb, 0x3a, 0xd8, 0x03, 0x9e, 0x06, 0x5e, 0x9a, 0x70, 0xfd, 0x31, + 0xd6, 0x75, 0x5b, 0x8a, 0x70, 0x94, 0x70, 0xa9, 0xa6, 0xa4, 0x31, 0x45, 0xda, 0x87, 0x7b, 0xff, + 0xb5, 0xa1, 0x73, 0x4c, 0xfd, 0x4b, 0x4a, 0x43, 0x4c, 0x56, 0x32, 0x28, 0x40, 0xb2, 0xfa, 0x8b, + 0x85, 0x3c, 0x1c, 0x47, 0xc3, 0xa9, 0xff, 0x74, 0xfa, 0x9f, 0xdd, 0x26, 0x96, 0xe3, 0xcd, 0x1c, + 0x39, 0x84, 0xb6, 0xf1, 0x0f, 0x83, 0x6c, 0x18, 0x07, 0x27, 0x7e, 0xcd, 0xf4, 0x37, 0x67, 0x70, + 0x0b, 0x6d, 0x4f, 0x2c, 0xf2, 0x06, 0xda, 0xc6, 0xa8, 0x6c, 0xea, 0x9b, 0x9c, 0xd9, 0x4d, 0x7d, + 0x53, 0xe6, 0x6b, 0x67, 0x4e, 0x69, 0x33, 0x06, 0x5e, 0x53, 0xdb, 0xe4, 0x88, 0x6d, 0x6a, 0x9b, + 0x36, 0x25, 0xcf, 0x11, 0x17, 0xba, 0x95, 0x21, 0x95, 0x6c, 0x8d, 0x4e, 0x4c, 0x9b, 0x9f, 0xfb, + 0xf7, 0x67, 0xf2, 0x4d, 0x0b, 0x8d, 0xb9, 0xd0, 0xb4, 0x70, 0x72, 0xea, 0x35, 0x2d, 0x9c, 0x32, + 0x4c, 0x3a, 0x73, 0xe4, 0x0f, 0xb0, 0x34, 0x31, 0x9b, 0x11, 0xc7, 0xb0, 0x62, 0xc6, 0x50, 0xd9, + 0x7f, 0x70, 0xa3, 0x4c, 0xa9, 0xff, 0x6b, 0xe8, 0x98, 0x23, 0x11, 0x31, 0x0c, 0x9a, 0x32, 0xf5, + 0xf5, 0xb7, 0x66, 0xb1, 0x4d, 0x85, 0x66, 0x57, 0x36, 0x15, 0x4e, 0x99, 0x4b, 0x4c, 0x85, 0xd3, + 0x9a, 0xb9, 0x33, 0x47, 0x7e, 0x0f, 0x8b, 0xe3, 0xdd, 0x91, 0x7c, 0x3a, 0xee, 0xb6, 0x89, 0xa6, + 0xdb, 0x77, 0x6e, 0x12, 0x29, 0x95, 0xbf, 0x02, 0x18, 0x35, 0x3d, 0x62, 0xc0, 0xcf, 0x44, 0xd3, + 0xed, 0x6f, 0x4c, 0x67, 0x96, 0xaa, 0xfe, 0x04, 0xab, 0x53, 0x3b, 0x0b, 0x31, 0x4a, 0xef, 0xa6, + 0xde, 0xd4, 0xff, 0xf1, 0xad, 0x72, 0xe5, 0x5d, 0x7f, 0x84, 0xa5, 0x09, 0xf8, 0x35, 0xb3, 0x62, + 0x56, 0x57, 0x30, 0xb3, 0x62, 0x26, 0x7e, 0x63, 0xd5, 0x7e, 0x03, 0xdd, 0x0a, 0x30, 0x99, 0x95, + 0x31, 0x0d, 0xfa, 0xcc, 0xca, 0x98, 0x8a, 0x68, 0xce, 0xdc, 0x8e, 0xf5, 0xc4, 0x7a, 0xb6, 0x05, + 0x8b, 0x42, 0xc3, 0xda, 0xa9, 0xd8, 0xd5, 0xdd, 0xe8, 0x19, 0xe0, 0x5b, 0x8f, 0x78, 0x22, 0x93, + 0x93, 0x26, 0xfe, 0xab, 0xfe, 0xd9, 0xff, 0x03, 0x00, 0x00, 0xff, 0xff, 0xf8, 0xd4, 0xa1, 0xe3, + 0xba, 0x16, 0x00, 0x00, } diff --git a/weed/pb/messaging.proto b/weed/pb/messaging.proto index 165542681..56f60b212 100644 --- a/weed/pb/messaging.proto +++ b/weed/pb/messaging.proto @@ -21,6 +21,9 @@ service SeaweedMessaging { rpc GetTopicConfiguration (GetTopicConfigurationRequest) returns (GetTopicConfigurationResponse) { } + rpc FindBroker (FindBrokerRequest) returns (FindBrokerResponse) { + } + } ////////////////////////////////////////////////// @@ -55,10 +58,7 @@ message Message { message BrokerMessage { Message data = 1; - message RedirectMessage { - string new_broker = 1; - } - RedirectMessage redirect = 2; + bool is_close = 2; } message PublishRequest { @@ -98,6 +98,16 @@ message GetTopicConfigurationResponse { TopicConfiguration configuration = 1; } +message FindBrokerRequest { + string namespace = 1; + string topic = 2; + int32 parition = 3; +} + +message FindBrokerResponse { + string broker = 1; +} + message TopicConfiguration { int32 partition_count = 1; string collection = 2; diff --git a/weed/pb/messaging_pb/messaging.pb.go b/weed/pb/messaging_pb/messaging.pb.go index 57f316708..809fd3ded 100644 --- a/weed/pb/messaging_pb/messaging.pb.go +++ b/weed/pb/messaging_pb/messaging.pb.go @@ -18,6 +18,8 @@ It has these top-level messages: ConfigureTopicResponse GetTopicConfigurationRequest GetTopicConfigurationResponse + FindBrokerRequest + FindBrokerResponse TopicConfiguration */ package messaging_pb @@ -91,7 +93,7 @@ func (x TopicConfiguration_Partitioning) String() string { return proto.EnumName(TopicConfiguration_Partitioning_name, int32(x)) } func (TopicConfiguration_Partitioning) EnumDescriptor() ([]byte, []int) { - return fileDescriptor0, []int{9, 0} + return fileDescriptor0, []int{11, 0} } type SubscriberMessage struct { @@ -233,8 +235,8 @@ func (m *Message) GetHeaders() map[string][]byte { } type BrokerMessage struct { - Data *Message `protobuf:"bytes,1,opt,name=data" json:"data,omitempty"` - Redirect *BrokerMessage_RedirectMessage `protobuf:"bytes,2,opt,name=redirect" json:"redirect,omitempty"` + Data *Message `protobuf:"bytes,1,opt,name=data" json:"data,omitempty"` + IsClose bool `protobuf:"varint,2,opt,name=is_close,json=isClose" json:"is_close,omitempty"` } func (m *BrokerMessage) Reset() { *m = BrokerMessage{} } @@ -249,29 +251,11 @@ func (m *BrokerMessage) GetData() *Message { return nil } -func (m *BrokerMessage) GetRedirect() *BrokerMessage_RedirectMessage { +func (m *BrokerMessage) GetIsClose() bool { if m != nil { - return m.Redirect + return m.IsClose } - return nil -} - -type BrokerMessage_RedirectMessage struct { - NewBroker string `protobuf:"bytes,1,opt,name=new_broker,json=newBroker" json:"new_broker,omitempty"` -} - -func (m *BrokerMessage_RedirectMessage) Reset() { *m = BrokerMessage_RedirectMessage{} } -func (m *BrokerMessage_RedirectMessage) String() string { return proto.CompactTextString(m) } -func (*BrokerMessage_RedirectMessage) ProtoMessage() {} -func (*BrokerMessage_RedirectMessage) Descriptor() ([]byte, []int) { - return fileDescriptor0, []int{2, 0} -} - -func (m *BrokerMessage_RedirectMessage) GetNewBroker() string { - if m != nil { - return m.NewBroker - } - return "" + return false } type PublishRequest struct { @@ -470,6 +454,54 @@ func (m *GetTopicConfigurationResponse) GetConfiguration() *TopicConfiguration { return nil } +type FindBrokerRequest struct { + Namespace string `protobuf:"bytes,1,opt,name=namespace" json:"namespace,omitempty"` + Topic string `protobuf:"bytes,2,opt,name=topic" json:"topic,omitempty"` + Parition int32 `protobuf:"varint,3,opt,name=parition" json:"parition,omitempty"` +} + +func (m *FindBrokerRequest) Reset() { *m = FindBrokerRequest{} } +func (m *FindBrokerRequest) String() string { return proto.CompactTextString(m) } +func (*FindBrokerRequest) ProtoMessage() {} +func (*FindBrokerRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{9} } + +func (m *FindBrokerRequest) GetNamespace() string { + if m != nil { + return m.Namespace + } + return "" +} + +func (m *FindBrokerRequest) GetTopic() string { + if m != nil { + return m.Topic + } + return "" +} + +func (m *FindBrokerRequest) GetParition() int32 { + if m != nil { + return m.Parition + } + return 0 +} + +type FindBrokerResponse struct { + Broker string `protobuf:"bytes,1,opt,name=broker" json:"broker,omitempty"` +} + +func (m *FindBrokerResponse) Reset() { *m = FindBrokerResponse{} } +func (m *FindBrokerResponse) String() string { return proto.CompactTextString(m) } +func (*FindBrokerResponse) ProtoMessage() {} +func (*FindBrokerResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{10} } + +func (m *FindBrokerResponse) GetBroker() string { + if m != nil { + return m.Broker + } + return "" +} + type TopicConfiguration struct { PartitionCount int32 `protobuf:"varint,1,opt,name=partition_count,json=partitionCount" json:"partition_count,omitempty"` Collection string `protobuf:"bytes,2,opt,name=collection" json:"collection,omitempty"` @@ -481,7 +513,7 @@ type TopicConfiguration struct { func (m *TopicConfiguration) Reset() { *m = TopicConfiguration{} } func (m *TopicConfiguration) String() string { return proto.CompactTextString(m) } func (*TopicConfiguration) ProtoMessage() {} -func (*TopicConfiguration) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{9} } +func (*TopicConfiguration) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{11} } func (m *TopicConfiguration) GetPartitionCount() int32 { if m != nil { @@ -524,7 +556,6 @@ func init() { proto.RegisterType((*SubscriberMessage_AckMessage)(nil), "messaging_pb.SubscriberMessage.AckMessage") proto.RegisterType((*Message)(nil), "messaging_pb.Message") proto.RegisterType((*BrokerMessage)(nil), "messaging_pb.BrokerMessage") - proto.RegisterType((*BrokerMessage_RedirectMessage)(nil), "messaging_pb.BrokerMessage.RedirectMessage") proto.RegisterType((*PublishRequest)(nil), "messaging_pb.PublishRequest") proto.RegisterType((*PublishRequest_InitMessage)(nil), "messaging_pb.PublishRequest.InitMessage") proto.RegisterType((*PublishResponse)(nil), "messaging_pb.PublishResponse") @@ -534,6 +565,8 @@ func init() { proto.RegisterType((*ConfigureTopicResponse)(nil), "messaging_pb.ConfigureTopicResponse") proto.RegisterType((*GetTopicConfigurationRequest)(nil), "messaging_pb.GetTopicConfigurationRequest") proto.RegisterType((*GetTopicConfigurationResponse)(nil), "messaging_pb.GetTopicConfigurationResponse") + proto.RegisterType((*FindBrokerRequest)(nil), "messaging_pb.FindBrokerRequest") + proto.RegisterType((*FindBrokerResponse)(nil), "messaging_pb.FindBrokerResponse") proto.RegisterType((*TopicConfiguration)(nil), "messaging_pb.TopicConfiguration") proto.RegisterEnum("messaging_pb.SubscriberMessage_InitMessage_StartPosition", SubscriberMessage_InitMessage_StartPosition_name, SubscriberMessage_InitMessage_StartPosition_value) proto.RegisterEnum("messaging_pb.TopicConfiguration_Partitioning", TopicConfiguration_Partitioning_name, TopicConfiguration_Partitioning_value) @@ -554,6 +587,7 @@ type SeaweedMessagingClient interface { Publish(ctx context.Context, opts ...grpc.CallOption) (SeaweedMessaging_PublishClient, error) ConfigureTopic(ctx context.Context, in *ConfigureTopicRequest, opts ...grpc.CallOption) (*ConfigureTopicResponse, error) GetTopicConfiguration(ctx context.Context, in *GetTopicConfigurationRequest, opts ...grpc.CallOption) (*GetTopicConfigurationResponse, error) + FindBroker(ctx context.Context, in *FindBrokerRequest, opts ...grpc.CallOption) (*FindBrokerResponse, error) } type seaweedMessagingClient struct { @@ -644,6 +678,15 @@ func (c *seaweedMessagingClient) GetTopicConfiguration(ctx context.Context, in * return out, nil } +func (c *seaweedMessagingClient) FindBroker(ctx context.Context, in *FindBrokerRequest, opts ...grpc.CallOption) (*FindBrokerResponse, error) { + out := new(FindBrokerResponse) + err := grpc.Invoke(ctx, "/messaging_pb.SeaweedMessaging/FindBroker", in, out, c.cc, opts...) + if err != nil { + return nil, err + } + return out, nil +} + // Server API for SeaweedMessaging service type SeaweedMessagingServer interface { @@ -651,6 +694,7 @@ type SeaweedMessagingServer interface { Publish(SeaweedMessaging_PublishServer) error ConfigureTopic(context.Context, *ConfigureTopicRequest) (*ConfigureTopicResponse, error) GetTopicConfiguration(context.Context, *GetTopicConfigurationRequest) (*GetTopicConfigurationResponse, error) + FindBroker(context.Context, *FindBrokerRequest) (*FindBrokerResponse, error) } func RegisterSeaweedMessagingServer(s *grpc.Server, srv SeaweedMessagingServer) { @@ -745,6 +789,24 @@ func _SeaweedMessaging_GetTopicConfiguration_Handler(srv interface{}, ctx contex return interceptor(ctx, in, info, handler) } +func _SeaweedMessaging_FindBroker_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(FindBrokerRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(SeaweedMessagingServer).FindBroker(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/messaging_pb.SeaweedMessaging/FindBroker", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(SeaweedMessagingServer).FindBroker(ctx, req.(*FindBrokerRequest)) + } + return interceptor(ctx, in, info, handler) +} + var _SeaweedMessaging_serviceDesc = grpc.ServiceDesc{ ServiceName: "messaging_pb.SeaweedMessaging", HandlerType: (*SeaweedMessagingServer)(nil), @@ -757,6 +819,10 @@ var _SeaweedMessaging_serviceDesc = grpc.ServiceDesc{ MethodName: "GetTopicConfiguration", Handler: _SeaweedMessaging_GetTopicConfiguration_Handler, }, + { + MethodName: "FindBroker", + Handler: _SeaweedMessaging_FindBroker_Handler, + }, }, Streams: []grpc.StreamDesc{ { @@ -778,62 +844,65 @@ var _SeaweedMessaging_serviceDesc = grpc.ServiceDesc{ func init() { proto.RegisterFile("messaging.proto", fileDescriptor0) } var fileDescriptor0 = []byte{ - // 898 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xb4, 0x56, 0x5d, 0x6f, 0xe3, 0x44, - 0x17, 0xee, 0x38, 0xe9, 0x47, 0x8e, 0x93, 0x34, 0xef, 0xd1, 0x5b, 0x14, 0x99, 0x16, 0x82, 0x17, - 0x41, 0xa0, 0xc2, 0xaa, 0xc2, 0x4d, 0x59, 0xad, 0xb4, 0x6a, 0xab, 0xb2, 0x1b, 0xd1, 0x76, 0xa3, - 0x49, 0x6e, 0x91, 0xe5, 0x38, 0xb3, 0xe9, 0xa8, 0xc9, 0x38, 0x78, 0x26, 0x5b, 0xf5, 0x1a, 0x6e, - 0xb9, 0xe2, 0xaf, 0xc0, 0x0f, 0xe0, 0x37, 0x70, 0xc7, 0xaf, 0x41, 0x1e, 0x7f, 0xc4, 0x4e, 0xb2, - 0xe9, 0x52, 0x89, 0x3b, 0xfb, 0xcc, 0x73, 0x9e, 0xf3, 0x9c, 0x2f, 0x8f, 0x61, 0x7f, 0xca, 0xa4, - 0xf4, 0xc6, 0x5c, 0x8c, 0x9d, 0x59, 0x18, 0xa8, 0x00, 0xab, 0x99, 0xc1, 0x9d, 0x0d, 0xed, 0x9f, - 0xcb, 0xf0, 0xbf, 0xfe, 0x7c, 0x28, 0xfd, 0x90, 0x0f, 0x59, 0x78, 0xad, 0x8f, 0x18, 0xbe, 0x84, - 0x32, 0x17, 0x5c, 0x35, 0x49, 0x8b, 0xb4, 0xcd, 0xce, 0xb1, 0x93, 0x77, 0x71, 0x56, 0xe0, 0x4e, - 0x57, 0x70, 0x95, 0x3c, 0x53, 0xed, 0x88, 0x2f, 0xa0, 0xe4, 0xf9, 0x77, 0x4d, 0x43, 0xfb, 0x7f, - 0xfd, 0x98, 0xff, 0x99, 0x7f, 0x97, 0xba, 0x47, 0x6e, 0xd6, 0x9f, 0x06, 0x98, 0x39, 0x4e, 0x3c, - 0x84, 0x8a, 0xf0, 0xa6, 0x4c, 0xce, 0x3c, 0x9f, 0x69, 0x4d, 0x15, 0xba, 0x30, 0xe0, 0xff, 0x61, - 0x5b, 0x05, 0x33, 0xee, 0xeb, 0x68, 0x15, 0x1a, 0xbf, 0x44, 0x3e, 0x33, 0x2f, 0x54, 0x5c, 0xf1, - 0x40, 0x34, 0x4b, 0x2d, 0xd2, 0xde, 0xa6, 0x0b, 0x03, 0xba, 0x50, 0x93, 0xca, 0x0b, 0x55, 0x2f, - 0x90, 0x31, 0xa2, 0xdc, 0x22, 0xed, 0x7a, 0xe7, 0xbb, 0x7f, 0x91, 0xa9, 0xd3, 0xcf, 0x13, 0xd0, - 0x22, 0x1f, 0xb6, 0xc0, 0x54, 0x7c, 0xca, 0xa4, 0xf2, 0xa6, 0xb3, 0x1b, 0xd9, 0xdc, 0x6e, 0x91, - 0x76, 0x89, 0xe6, 0x4d, 0xf8, 0x0c, 0x6a, 0x32, 0xe3, 0x77, 0xf9, 0xa8, 0xb9, 0xa3, 0xe5, 0x57, - 0x17, 0xc6, 0xee, 0xc8, 0x3e, 0x85, 0x5a, 0x21, 0x0c, 0x02, 0xec, 0x5c, 0x9d, 0x0d, 0x2e, 0xfb, - 0x83, 0xc6, 0x16, 0x56, 0x61, 0xef, 0xf2, 0x8c, 0x5e, 0x75, 0xa3, 0x37, 0x82, 0x35, 0xa8, 0x0c, - 0xba, 0xd7, 0x97, 0xfd, 0xc1, 0xd9, 0x75, 0xaf, 0x61, 0x58, 0xc7, 0x00, 0x8b, 0xb2, 0xe2, 0x11, - 0x40, 0x9c, 0x19, 0x8b, 0x22, 0x11, 0xad, 0xa6, 0x92, 0x58, 0xba, 0x23, 0xfb, 0x2f, 0x02, 0xbb, - 0x29, 0xf4, 0x0b, 0xa8, 0xb1, 0x77, 0x4c, 0x28, 0x37, 0x12, 0xeb, 0x0a, 0x19, 0xa3, 0xcf, 0x8d, - 0x13, 0x42, 0x4d, 0x7d, 0x30, 0xe0, 0x53, 0x76, 0x23, 0xb1, 0x01, 0xa5, 0x3b, 0xf6, 0xa0, 0x8b, - 0x5e, 0xa5, 0xd1, 0x63, 0xd4, 0x88, 0x77, 0xde, 0x64, 0xce, 0x74, 0xb9, 0xab, 0x34, 0x7e, 0xc1, - 0x17, 0xb0, 0x7b, 0xcb, 0xbc, 0x11, 0x0b, 0x65, 0xb3, 0xdc, 0x2a, 0xb5, 0xcd, 0x8e, 0x5d, 0x2c, - 0x72, 0x5a, 0xce, 0xd7, 0x31, 0xe8, 0x52, 0xa8, 0xf0, 0x81, 0xa6, 0x2e, 0xd6, 0x73, 0xa8, 0xe6, - 0x0f, 0xd2, 0xa8, 0xf1, 0x10, 0x14, 0xa3, 0x1a, 0xb9, 0xa8, 0xcf, 0x8d, 0x53, 0x62, 0xff, 0x41, - 0xa0, 0x76, 0x1e, 0x06, 0x77, 0x8b, 0xb9, 0xfe, 0x0a, 0xca, 0x23, 0x4f, 0x79, 0xc9, 0x5c, 0x1f, - 0xac, 0x15, 0x42, 0x35, 0x04, 0x5f, 0xc1, 0x5e, 0xc8, 0x46, 0x3c, 0x64, 0xbe, 0x4a, 0xc6, 0x78, - 0x69, 0x0d, 0x0a, 0xcc, 0x0e, 0x4d, 0xb0, 0x29, 0x49, 0xe6, 0x6c, 0x9d, 0xc0, 0xfe, 0xd2, 0x61, - 0xd4, 0x0d, 0xc1, 0xee, 0xdd, 0xa1, 0x66, 0xc8, 0x06, 0x9a, 0xdd, 0xc7, 0x94, 0xf6, 0xdf, 0x04, - 0xea, 0xbd, 0xf9, 0x70, 0xc2, 0xe5, 0x2d, 0x65, 0x3f, 0xcd, 0x99, 0x8c, 0xf6, 0x29, 0xbf, 0x90, - 0xed, 0xa2, 0x92, 0x22, 0x76, 0xcd, 0x36, 0xa6, 0x69, 0x1b, 0x8f, 0xa6, 0x6d, 0xb9, 0xff, 0xf1, - 0xe6, 0xd9, 0xbf, 0x1a, 0xb0, 0x9f, 0x09, 0x96, 0xb3, 0x40, 0x48, 0x86, 0x17, 0xb0, 0xe3, 0x07, - 0xe2, 0x2d, 0x1f, 0xaf, 0xff, 0xe0, 0x2c, 0xc1, 0x9d, 0x0b, 0x8d, 0x4d, 0x75, 0x27, 0xae, 0xd8, - 0x5d, 0x69, 0xd8, 0x37, 0x9b, 0x69, 0xde, 0xdf, 0xb2, 0x53, 0xa8, 0x15, 0x62, 0xe0, 0x97, 0xb0, - 0x9f, 0x65, 0xe0, 0xfa, 0xc1, 0x5c, 0xc4, 0x9d, 0xd8, 0xa6, 0xf5, 0xcc, 0x7c, 0x11, 0x59, 0x9f, - 0xd0, 0xec, 0xdf, 0x08, 0x1c, 0xc4, 0xc1, 0xe6, 0x21, 0x1b, 0x44, 0x05, 0x4c, 0x7b, 0xfe, 0x94, - 0xda, 0x7f, 0x0f, 0x35, 0x3f, 0x21, 0xf3, 0xb2, 0xfa, 0x9b, 0x9d, 0x56, 0xb1, 0x12, 0x3a, 0xcc, - 0x45, 0x1e, 0x47, 0x8b, 0x6e, 0x76, 0x13, 0x3e, 0x5a, 0x16, 0x15, 0x57, 0xcd, 0xa6, 0x70, 0xf8, - 0x8a, 0xa9, 0x35, 0x0c, 0x4f, 0x57, 0x6d, 0x8f, 0xe1, 0xe8, 0x3d, 0x9c, 0xc9, 0x80, 0xac, 0xa4, - 0x45, 0x9e, 0x96, 0xd6, 0xef, 0x06, 0xe0, 0x2a, 0xea, 0x83, 0xdb, 0x8b, 0x9f, 0x00, 0xf8, 0xc1, - 0x64, 0xc2, 0x7c, 0x2d, 0x22, 0xce, 0x21, 0x67, 0x89, 0xbe, 0xfa, 0x21, 0x9b, 0x4d, 0xb8, 0xbf, - 0x28, 0x7e, 0x85, 0xe6, 0x4d, 0xf8, 0x19, 0x54, 0xb9, 0x74, 0x55, 0xe8, 0x09, 0xc9, 0x99, 0x50, - 0xfa, 0xde, 0xd9, 0xa3, 0x26, 0x97, 0x83, 0xd4, 0x84, 0x6f, 0xc0, 0x8c, 0xc3, 0x06, 0x82, 0x8b, - 0xb1, 0xbe, 0x3a, 0xea, 0xcb, 0xb3, 0xbc, 0x9a, 0x84, 0xd3, 0x4b, 0xa5, 0x72, 0x31, 0xa6, 0x79, - 0x06, 0xfb, 0x25, 0x54, 0xf3, 0x87, 0x88, 0x50, 0xbf, 0x09, 0xc4, 0xcd, 0x7c, 0x32, 0xf9, 0x81, - 0x3d, 0xbc, 0xf6, 0xe4, 0x6d, 0x63, 0x0b, 0x4d, 0xd8, 0x4d, 0x5f, 0x08, 0xd6, 0x01, 0x68, 0x30, - 0x17, 0x23, 0x1a, 0x0c, 0xb9, 0x68, 0x18, 0x9d, 0x5f, 0x4a, 0xd0, 0xe8, 0x33, 0xef, 0x9e, 0xb1, - 0xd1, 0x75, 0xaa, 0x02, 0xdf, 0x40, 0x25, 0xbb, 0x1f, 0xf1, 0xd3, 0x47, 0x2e, 0x4e, 0xeb, 0xe3, - 0x0d, 0x1f, 0x4f, 0x7b, 0xab, 0x4d, 0x4e, 0x08, 0x5e, 0xc1, 0x6e, 0xb2, 0xa2, 0x78, 0xb8, 0xe9, - 0x03, 0x67, 0x1d, 0x6d, 0xdc, 0xeb, 0x84, 0xed, 0x47, 0xa8, 0x17, 0x27, 0x18, 0x9f, 0x15, 0xdd, - 0xd6, 0x2e, 0x9d, 0xf5, 0xf9, 0x66, 0x50, 0x1a, 0x02, 0x43, 0x38, 0x58, 0x3b, 0xb2, 0xb8, 0xf4, - 0xb3, 0xb3, 0x69, 0x57, 0xac, 0xe3, 0x0f, 0xc2, 0xa6, 0x31, 0xcf, 0x6d, 0x68, 0xc8, 0xb8, 0x0b, - 0x6f, 0xa5, 0xe3, 0x4f, 0xa2, 0x61, 0x39, 0xaf, 0x67, 0x0d, 0xe9, 0x45, 0x7f, 0x77, 0xc3, 0x1d, - 0xfd, 0x93, 0xf7, 0xed, 0x3f, 0x01, 0x00, 0x00, 0xff, 0xff, 0x3b, 0xcc, 0x19, 0xa5, 0xf7, 0x09, - 0x00, 0x00, + // 952 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xb4, 0x56, 0xdd, 0x6e, 0xe3, 0x44, + 0x14, 0xae, 0x9d, 0x34, 0x3f, 0x27, 0x3f, 0xcd, 0x1e, 0xd1, 0x55, 0x30, 0x2d, 0x04, 0x2f, 0x82, + 0x40, 0x21, 0xaa, 0xc2, 0x4d, 0x59, 0xad, 0xb4, 0x6a, 0xa3, 0x2e, 0x1b, 0xd1, 0x76, 0xc3, 0x24, + 0x5c, 0x22, 0xcb, 0xb1, 0x67, 0xd3, 0x51, 0x9d, 0xb1, 0xf1, 0x38, 0x5b, 0xf5, 0x9a, 0x6b, 0xae, + 0x78, 0x15, 0x5e, 0x80, 0x67, 0xe0, 0x02, 0x89, 0xa7, 0x41, 0x1e, 0xff, 0xc4, 0x4e, 0xb2, 0xe9, + 0x52, 0xb4, 0x77, 0x9e, 0x33, 0xdf, 0xf9, 0xce, 0x77, 0xce, 0x9c, 0x33, 0x63, 0xd8, 0x9b, 0x53, + 0x21, 0xcc, 0x19, 0xe3, 0xb3, 0x9e, 0xe7, 0xbb, 0x81, 0x8b, 0xf5, 0xd4, 0x60, 0x78, 0x53, 0xfd, + 0xd7, 0x22, 0x3c, 0x1a, 0x2f, 0xa6, 0xc2, 0xf2, 0xd9, 0x94, 0xfa, 0x97, 0x72, 0x8b, 0xe2, 0x73, + 0x28, 0x32, 0xce, 0x82, 0xb6, 0xd2, 0x51, 0xba, 0xb5, 0xfe, 0x51, 0x2f, 0xeb, 0xd2, 0x5b, 0x83, + 0xf7, 0x86, 0x9c, 0x05, 0xf1, 0x37, 0x91, 0x8e, 0xf8, 0x0c, 0x0a, 0xa6, 0x75, 0xd3, 0x56, 0xa5, + 0xff, 0x57, 0xf7, 0xf9, 0x9f, 0x5a, 0x37, 0x89, 0x7b, 0xe8, 0xa6, 0xfd, 0xa9, 0x42, 0x2d, 0xc3, + 0x89, 0x07, 0x50, 0xe5, 0xe6, 0x9c, 0x0a, 0xcf, 0xb4, 0xa8, 0xd4, 0x54, 0x25, 0x4b, 0x03, 0x7e, + 0x00, 0xbb, 0x81, 0xeb, 0x31, 0x4b, 0x46, 0xab, 0x92, 0x68, 0x11, 0xfa, 0x78, 0xa6, 0x1f, 0xb0, + 0x80, 0xb9, 0xbc, 0x5d, 0xe8, 0x28, 0xdd, 0x5d, 0xb2, 0x34, 0xa0, 0x01, 0x0d, 0x11, 0x98, 0x7e, + 0x30, 0x72, 0x45, 0x84, 0x28, 0x76, 0x94, 0x6e, 0xb3, 0xff, 0xdd, 0x7f, 0xc8, 0xb4, 0x37, 0xce, + 0x12, 0x90, 0x3c, 0x1f, 0x76, 0xa0, 0x16, 0xb0, 0x39, 0x15, 0x81, 0x39, 0xf7, 0xae, 0x44, 0x7b, + 0xb7, 0xa3, 0x74, 0x0b, 0x24, 0x6b, 0xc2, 0x27, 0xd0, 0x10, 0x29, 0xbf, 0xc1, 0xec, 0x76, 0x49, + 0xca, 0xaf, 0x2f, 0x8d, 0x43, 0x5b, 0x3f, 0x81, 0x46, 0x2e, 0x0c, 0x02, 0x94, 0x2e, 0x4e, 0x27, + 0xe7, 0xe3, 0x49, 0x6b, 0x07, 0xeb, 0x50, 0x39, 0x3f, 0x25, 0x17, 0xc3, 0x70, 0xa5, 0x60, 0x03, + 0xaa, 0x93, 0xe1, 0xe5, 0xf9, 0x78, 0x72, 0x7a, 0x39, 0x6a, 0xa9, 0xda, 0x11, 0xc0, 0xb2, 0xac, + 0x78, 0x08, 0x10, 0x65, 0x46, 0xc3, 0x48, 0x8a, 0x54, 0x53, 0x8d, 0x2d, 0x43, 0x5b, 0xff, 0x4b, + 0x81, 0x72, 0x02, 0xfd, 0x1c, 0x1a, 0xf4, 0x0d, 0xe5, 0x81, 0x11, 0x8a, 0x35, 0xb8, 0x88, 0xd0, + 0x67, 0xea, 0xb1, 0x42, 0x6a, 0x72, 0x63, 0xc2, 0xe6, 0xf4, 0x4a, 0x60, 0x0b, 0x0a, 0x37, 0xf4, + 0x4e, 0x16, 0xbd, 0x4e, 0xc2, 0xcf, 0xf0, 0x20, 0xde, 0x98, 0xce, 0x82, 0xca, 0x72, 0xd7, 0x49, + 0xb4, 0xc0, 0x67, 0x50, 0xbe, 0xa6, 0xa6, 0x4d, 0x7d, 0xd1, 0x2e, 0x76, 0x0a, 0xdd, 0x5a, 0x5f, + 0xcf, 0x17, 0x39, 0x29, 0xe7, 0xcb, 0x08, 0x74, 0xce, 0x03, 0xff, 0x8e, 0x24, 0x2e, 0xda, 0x53, + 0xa8, 0x67, 0x37, 0x92, 0xa8, 0x51, 0x13, 0xe4, 0xa3, 0xaa, 0x99, 0xa8, 0x4f, 0xd5, 0x13, 0x45, + 0xff, 0x09, 0x1a, 0x67, 0xbe, 0x7b, 0xb3, 0x6c, 0xeb, 0x2f, 0xa1, 0x68, 0x9b, 0x81, 0x19, 0xb7, + 0xf5, 0xfe, 0x46, 0x1d, 0x44, 0x42, 0xf0, 0x43, 0xa8, 0x30, 0x61, 0x58, 0x8e, 0x2b, 0x22, 0xe2, + 0x0a, 0x29, 0x33, 0x31, 0x08, 0x97, 0xfa, 0x3f, 0x0a, 0x34, 0x47, 0x8b, 0xa9, 0xc3, 0xc4, 0x35, + 0xa1, 0xbf, 0x2c, 0xa8, 0x08, 0xdb, 0x3d, 0x3b, 0x2f, 0xdd, 0x3c, 0x71, 0x1e, 0xbb, 0x61, 0x58, + 0x12, 0x59, 0xea, 0xbd, 0xb2, 0x34, 0xe3, 0x3d, 0x0f, 0x86, 0xfe, 0x9b, 0x0a, 0x7b, 0xa9, 0x60, + 0xe1, 0xb9, 0x5c, 0x50, 0x1c, 0x40, 0xc9, 0x72, 0xf9, 0x6b, 0x36, 0xdb, 0x7c, 0x1f, 0xac, 0xc0, + 0x7b, 0x03, 0x89, 0x4d, 0x74, 0xc7, 0xae, 0x38, 0x84, 0x8a, 0x4f, 0x6d, 0xe6, 0x53, 0x2b, 0x88, + 0x13, 0xfd, 0x66, 0x3b, 0x0d, 0x89, 0xd1, 0x09, 0x51, 0xea, 0xae, 0x9d, 0x40, 0x23, 0x17, 0x03, + 0xbf, 0x80, 0xbd, 0x34, 0x03, 0xc3, 0x72, 0x17, 0x3c, 0x3a, 0x89, 0x5d, 0xd2, 0x4c, 0xcd, 0x83, + 0xd0, 0xaa, 0x1d, 0xc3, 0xde, 0x0a, 0x6d, 0x38, 0x19, 0x9c, 0xde, 0x1a, 0x53, 0xd9, 0x28, 0x69, + 0x0d, 0xe9, 0x6d, 0xd4, 0x39, 0xfa, 0xef, 0x0a, 0xec, 0x47, 0xc1, 0x16, 0x3e, 0x9d, 0x84, 0x05, + 0x4c, 0xce, 0xfc, 0x21, 0xb5, 0x7f, 0x01, 0x0d, 0x2b, 0x26, 0x33, 0xd3, 0xfa, 0xd7, 0xfa, 0x9d, + 0x7c, 0x25, 0x64, 0x98, 0x41, 0x16, 0x47, 0xf2, 0x6e, 0x7a, 0x1b, 0x1e, 0xaf, 0x8a, 0x8a, 0xaa, + 0xa6, 0x13, 0x38, 0xf8, 0x9e, 0x06, 0x1b, 0x18, 0x1e, 0xae, 0x5a, 0x9f, 0xc1, 0xe1, 0x5b, 0x38, + 0xe3, 0x06, 0x59, 0x4b, 0x4b, 0x79, 0x58, 0x5a, 0x16, 0x3c, 0x7a, 0xc1, 0xb8, 0x1d, 0x95, 0xfe, + 0xff, 0xd4, 0x59, 0x83, 0x8a, 0x67, 0xfa, 0xd9, 0x16, 0x4f, 0xd7, 0xfa, 0xd7, 0x80, 0xd9, 0x20, + 0x71, 0x0a, 0x8f, 0xa1, 0x94, 0x6b, 0x81, 0x78, 0xa5, 0xff, 0xa1, 0x02, 0xae, 0x0b, 0x7f, 0xe7, + 0x8e, 0xc3, 0x8f, 0x01, 0x2c, 0xd7, 0x71, 0xa8, 0x25, 0xb5, 0x44, 0x22, 0x33, 0x96, 0xf0, 0x9d, + 0xf0, 0xa9, 0xe7, 0x30, 0x6b, 0xd9, 0x0f, 0x55, 0x92, 0x35, 0xe1, 0xa7, 0x50, 0x67, 0xc2, 0x08, + 0x7c, 0x93, 0x0b, 0x46, 0x79, 0x20, 0x5f, 0xaa, 0x0a, 0xa9, 0x31, 0x31, 0x49, 0x4c, 0xf8, 0x0a, + 0x6a, 0x51, 0x58, 0x97, 0x33, 0x3e, 0x93, 0x8f, 0x4d, 0x73, 0x75, 0xbc, 0xd6, 0x93, 0xe8, 0x8d, + 0x12, 0xa9, 0x8c, 0xcf, 0x48, 0x96, 0x41, 0x7f, 0x0e, 0xf5, 0xec, 0x26, 0x22, 0x34, 0xaf, 0x5c, + 0x7e, 0xb5, 0x70, 0x9c, 0x1f, 0xe8, 0xdd, 0x4b, 0x53, 0x5c, 0xb7, 0x76, 0xb0, 0x06, 0xe5, 0x64, + 0xa1, 0x60, 0x13, 0x80, 0xb8, 0x0b, 0x6e, 0x13, 0x77, 0xca, 0x78, 0x4b, 0xed, 0xff, 0x5d, 0x80, + 0xd6, 0x98, 0x9a, 0xb7, 0x94, 0xda, 0x97, 0x89, 0x0a, 0x7c, 0x05, 0xd5, 0xf4, 0x45, 0xc5, 0x4f, + 0xee, 0x79, 0x6a, 0xb5, 0x8f, 0xf2, 0x80, 0xdc, 0x4d, 0xae, 0xef, 0x74, 0x95, 0x63, 0x05, 0x2f, + 0xa0, 0x1c, 0xdf, 0x1a, 0x78, 0xb0, 0xed, 0xce, 0xd5, 0x0e, 0xb7, 0x5e, 0x35, 0x31, 0xdb, 0xcf, + 0xd0, 0xcc, 0x0f, 0x15, 0x3e, 0xc9, 0xbb, 0x6d, 0xbc, 0x07, 0xb4, 0xcf, 0xb6, 0x83, 0x92, 0x10, + 0xe8, 0xc3, 0xfe, 0xc6, 0x29, 0xc2, 0x95, 0xdf, 0xa3, 0x6d, 0xe3, 0xab, 0x1d, 0xbd, 0x13, 0x36, + 0x8d, 0xf9, 0x23, 0xc0, 0xb2, 0xd7, 0x57, 0x4b, 0xbe, 0x36, 0x6a, 0x5a, 0xe7, 0xed, 0x80, 0x84, + 0xf2, 0x4c, 0x87, 0x96, 0x88, 0x0e, 0xf6, 0xb5, 0xe8, 0x59, 0x4e, 0xd8, 0x7f, 0x67, 0xcd, 0xf4, + 0x8c, 0x47, 0xe1, 0x2f, 0xe6, 0xb4, 0x24, 0xff, 0x34, 0xbf, 0xfd, 0x37, 0x00, 0x00, 0xff, 0xff, + 0xad, 0x6b, 0x26, 0x8c, 0x7c, 0x0a, 0x00, 0x00, } diff --git a/weed/pb/volume_server_pb/volume_server.pb.go b/weed/pb/volume_server_pb/volume_server.pb.go index 8b02d2bba..ca96b8d20 100644 --- a/weed/pb/volume_server_pb/volume_server.pb.go +++ b/weed/pb/volume_server_pb/volume_server.pb.go @@ -1051,12 +1051,10 @@ func (m *VolumeEcShardsGenerateRequest) GetCollection() string { type VolumeEcShardsGenerateResponse struct { } -func (m *VolumeEcShardsGenerateResponse) Reset() { *m = VolumeEcShardsGenerateResponse{} } -func (m *VolumeEcShardsGenerateResponse) String() string { return proto.CompactTextString(m) } -func (*VolumeEcShardsGenerateResponse) ProtoMessage() {} -func (*VolumeEcShardsGenerateResponse) Descriptor() ([]byte, []int) { - return fileDescriptor0, []int{41} -} +func (m *VolumeEcShardsGenerateResponse) Reset() { *m = VolumeEcShardsGenerateResponse{} } +func (m *VolumeEcShardsGenerateResponse) String() string { return proto.CompactTextString(m) } +func (*VolumeEcShardsGenerateResponse) ProtoMessage() {} +func (*VolumeEcShardsGenerateResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{41} } type VolumeEcShardsRebuildRequest struct { VolumeId uint32 `protobuf:"varint,1,opt,name=volume_id,json=volumeId" json:"volume_id,omitempty"` @@ -1429,12 +1427,10 @@ func (m *VolumeEcShardsToVolumeRequest) GetCollection() string { type VolumeEcShardsToVolumeResponse struct { } -func (m *VolumeEcShardsToVolumeResponse) Reset() { *m = VolumeEcShardsToVolumeResponse{} } -func (m *VolumeEcShardsToVolumeResponse) String() string { return proto.CompactTextString(m) } -func (*VolumeEcShardsToVolumeResponse) ProtoMessage() {} -func (*VolumeEcShardsToVolumeResponse) Descriptor() ([]byte, []int) { - return fileDescriptor0, []int{57} -} +func (m *VolumeEcShardsToVolumeResponse) Reset() { *m = VolumeEcShardsToVolumeResponse{} } +func (m *VolumeEcShardsToVolumeResponse) String() string { return proto.CompactTextString(m) } +func (*VolumeEcShardsToVolumeResponse) ProtoMessage() {} +func (*VolumeEcShardsToVolumeResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{57} } type ReadVolumeFileStatusRequest struct { VolumeId uint32 `protobuf:"varint,1,opt,name=volume_id,json=volumeId" json:"volume_id,omitempty"` @@ -2101,10 +2097,8 @@ type QueryRequest_InputSerialization_JSONInput struct { func (m *QueryRequest_InputSerialization_JSONInput) Reset() { *m = QueryRequest_InputSerialization_JSONInput{} } -func (m *QueryRequest_InputSerialization_JSONInput) String() string { - return proto.CompactTextString(m) -} -func (*QueryRequest_InputSerialization_JSONInput) ProtoMessage() {} +func (m *QueryRequest_InputSerialization_JSONInput) String() string { return proto.CompactTextString(m) } +func (*QueryRequest_InputSerialization_JSONInput) ProtoMessage() {} func (*QueryRequest_InputSerialization_JSONInput) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{70, 1, 1} } diff --git a/weed/server/filer_grpc_server.go b/weed/server/filer_grpc_server.go index a790e4601..266030f5d 100644 --- a/weed/server/filer_grpc_server.go +++ b/weed/server/filer_grpc_server.go @@ -381,3 +381,39 @@ func (fs *FilerServer) GetFilerConfiguration(ctx context.Context, req *filer_pb. return t, nil } + +func (fs *FilerServer) KeepConnected(stream filer_pb.SeaweedFiler_KeepConnectedServer) error { + + req, err := stream.Recv() + if err != nil { + return err + } + + clientName := fmt.Sprintf("%s:%d", req.Name, req.GrpcPort) + fs.brokersLock.Lock() + fs.brokers[clientName] = true + glog.V(0).Infof("+ broker %v", clientName) + fs.brokersLock.Unlock() + + defer func() { + fs.brokersLock.Lock() + delete(fs.brokers, clientName) + glog.V(0).Infof("- broker %v: %v", clientName, err) + fs.brokersLock.Unlock() + }() + + for { + if err := stream.Send(&filer_pb.KeepConnectedResponse{}); err != nil { + glog.V(0).Infof("send broker %v: %+v", clientName, err) + return err + } + // println("replied") + + if _, err := stream.Recv(); err != nil { + glog.V(0).Infof("recv broker %v: %v", clientName, err) + return err + } + // println("received") + } + +} diff --git a/weed/server/filer_server.go b/weed/server/filer_server.go index 956684d46..0a54ea97d 100644 --- a/weed/server/filer_server.go +++ b/weed/server/filer_server.go @@ -8,9 +8,10 @@ import ( "sync" "time" - "github.com/chrislusf/seaweedfs/weed/util/grace" "google.golang.org/grpc" + "github.com/chrislusf/seaweedfs/weed/util/grace" + "github.com/chrislusf/seaweedfs/weed/operation" "github.com/chrislusf/seaweedfs/weed/pb" "github.com/chrislusf/seaweedfs/weed/pb/master_pb" @@ -62,6 +63,9 @@ type FilerServer struct { // notifying clients listenersLock sync.Mutex listenersCond *sync.Cond + + brokers map[string]bool + brokersLock sync.Mutex } func NewFilerServer(defaultMux, readonlyMux *http.ServeMux, option *FilerOption) (fs *FilerServer, err error) { @@ -69,6 +73,7 @@ func NewFilerServer(defaultMux, readonlyMux *http.ServeMux, option *FilerOption) fs = &FilerServer{ option: option, grpcDialOption: security.LoadClientTLS(util.GetViper(), "grpc.filer"), + brokers: make(map[string]bool), } fs.listenersCond = sync.NewCond(&fs.listenersLock) From dfccc3c2637693dce141c27a321ba5d3aea1ace9 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Fri, 8 May 2020 02:47:22 -0700 Subject: [PATCH 02/24] able to read chan and write chan --- go.mod | 2 + go.sum | 2 + other/java/client/src/main/proto/filer.proto | 19 + weed/messaging/broker/broker_append.go | 4 + weed/messaging/broker/broker_grpc_server.go | 4 + .../broker/broker_grpc_server_discovery.go | 35 +- .../broker/broker_grpc_server_publish.go | 30 +- .../broker/broker_grpc_server_subscribe.go | 12 +- weed/messaging/broker/broker_server.go | 24 +- .../broker/consistent_distribution.go | 38 ++ .../broker/consistent_distribution_test.go | 32 ++ weed/messaging/broker/topic_lock.go | 17 + weed/messaging/msgclient/pub_sub_chan.go | 38 +- weed/messaging/msgclient/publisher.go | 11 +- weed/messaging/msgclient/subscriber.go | 12 +- weed/pb/filer.proto | 19 + weed/pb/filer_pb/filer.pb.go | 381 ++++++++++++------ weed/pb/messaging.proto | 14 +- weed/pb/messaging_pb/messaging.pb.go | 248 ++++++++---- weed/server/filer_grpc_server.go | 38 +- weed/server/filer_grpc_server_listen.go | 2 +- weed/server/filer_server.go | 4 +- weed/util/log_buffer/log_read.go | 6 +- 23 files changed, 733 insertions(+), 259 deletions(-) create mode 100644 weed/messaging/broker/consistent_distribution.go create mode 100644 weed/messaging/broker/consistent_distribution_test.go diff --git a/go.mod b/go.mod index 96ea8fd3d..2ba8658d3 100644 --- a/go.mod +++ b/go.mod @@ -10,6 +10,8 @@ require ( github.com/OneOfOne/xxhash v1.2.2 github.com/Shopify/sarama v1.23.1 github.com/aws/aws-sdk-go v1.23.13 + github.com/buraksezer/consistent v0.0.0-20191006190839-693edf70fd72 + github.com/cespare/xxhash v1.1.0 github.com/chrislusf/raft v0.0.0-20190225081310-10d6e2182d92 github.com/coreos/bbolt v1.3.3 // indirect github.com/coreos/etcd v3.3.15+incompatible // indirect diff --git a/go.sum b/go.sum index a382daf10..00329907e 100644 --- a/go.sum +++ b/go.sum @@ -58,6 +58,8 @@ github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932 h1:mXoPYz/Ul5HYE github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932/go.mod h1:NOuUCSz6Q9T7+igc/hlvDOUdtWKryOrtFyIVABv/p7k= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= +github.com/buraksezer/consistent v0.0.0-20191006190839-693edf70fd72 h1:fUmDBbSvv1uOzo/t8WaxZMVb7BxJ8JECo5lGoR9c5bA= +github.com/buraksezer/consistent v0.0.0-20191006190839-693edf70fd72/go.mod h1:OEE5igu/CDjGegM1Jn6ZMo7R6LlV/JChAkjfQQIRLpg= github.com/census-instrumentation/opencensus-proto v0.2.0 h1:LzQXZOgg4CQfE6bFvXGM30YZL1WW/M337pXml+GrcZ4= github.com/census-instrumentation/opencensus-proto v0.2.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/census-instrumentation/opencensus-proto v0.2.1 h1:glEXhBS5PSLLv4IXzLA5yPRVX4bilULVyxxbrfOtDAk= diff --git a/other/java/client/src/main/proto/filer.proto b/other/java/client/src/main/proto/filer.proto index 3b3b78bbb..1fc8ef63d 100644 --- a/other/java/client/src/main/proto/filer.proto +++ b/other/java/client/src/main/proto/filer.proto @@ -50,6 +50,10 @@ service SeaweedFiler { rpc KeepConnected (stream KeepConnectedRequest) returns (stream KeepConnectedResponse) { } + + rpc LocateBroker (LocateBrokerRequest) returns (LocateBrokerResponse) { + } + } ////////////////////////////////////////////////// @@ -267,6 +271,21 @@ message LogEntry { message KeepConnectedRequest { string name = 1; uint32 grpc_port = 2; + repeated string resources = 3; } message KeepConnectedResponse { } + +message LocateBrokerRequest { + string resource = 1; +} +message LocateBrokerResponse { + bool found = 1; + // if found, send the exact address + // if not found, send the full list of existing brokers + message Resource { + string grpc_addresses = 1; + int32 resource_count = 2; + } + repeated Resource resources = 2; +} diff --git a/weed/messaging/broker/broker_append.go b/weed/messaging/broker/broker_append.go index e87e197b0..80f107e00 100644 --- a/weed/messaging/broker/broker_append.go +++ b/weed/messaging/broker/broker_append.go @@ -3,6 +3,7 @@ package broker import ( "context" "fmt" + "io" "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/operation" @@ -94,6 +95,9 @@ func (broker *MessageBroker) WithFilerClient(fn func(filer_pb.SeaweedFilerClient for _, filer := range broker.option.Filers { if err = pb.WithFilerClient(filer, broker.grpcDialOption, fn); err != nil { + if err == io.EOF { + return + } glog.V(0).Infof("fail to connect to %s: %v", filer, err) } else { break diff --git a/weed/messaging/broker/broker_grpc_server.go b/weed/messaging/broker/broker_grpc_server.go index 447620a6b..32dab6813 100644 --- a/weed/messaging/broker/broker_grpc_server.go +++ b/weed/messaging/broker/broker_grpc_server.go @@ -10,6 +10,10 @@ func (broker *MessageBroker) ConfigureTopic(c context.Context, request *messagin panic("implement me") } +func (broker *MessageBroker) DeleteTopic(c context.Context, request *messaging_pb.DeleteTopicRequest) (*messaging_pb.DeleteTopicResponse, error) { + panic("implement me") +} + func (broker *MessageBroker) GetTopicConfiguration(c context.Context, request *messaging_pb.GetTopicConfigurationRequest) (*messaging_pb.GetTopicConfigurationResponse, error) { panic("implement me") } diff --git a/weed/messaging/broker/broker_grpc_server_discovery.go b/weed/messaging/broker/broker_grpc_server_discovery.go index 4b7f357fa..3c14f3220 100644 --- a/weed/messaging/broker/broker_grpc_server_discovery.go +++ b/weed/messaging/broker/broker_grpc_server_discovery.go @@ -2,6 +2,7 @@ package broker import ( "context" + "fmt" "time" "github.com/chrislusf/seaweedfs/weed/glog" @@ -25,12 +26,40 @@ If one of the pub or sub connects very late, and the system topo changed quite a func (broker *MessageBroker) FindBroker(c context.Context, request *messaging_pb.FindBrokerRequest) (*messaging_pb.FindBrokerResponse, error) { - panic("implement me") -} + t := &messaging_pb.FindBrokerResponse{} + var peers []string + + targetTopicPartition := fmt.Sprintf(TopicPartitionFmt, request.Namespace, request.Topic, request.Parition) + + for _, filer := range broker.option.Filers { + err := broker.withFilerClient(filer, func(client filer_pb.SeaweedFilerClient) error { + resp, err := client.LocateBroker(context.Background(), &filer_pb.LocateBrokerRequest{ + Resource: targetTopicPartition, + }) + if err != nil { + return err + } + if resp.Found && len(resp.Resources) > 0 { + t.Broker = resp.Resources[0].GrpcAddresses + return nil + } + for _, b := range resp.Resources { + peers = append(peers, b.GrpcAddresses) + } + return nil + }) + if err != nil { + return nil, err + } + } + + t.Broker = PickMember(peers, []byte(targetTopicPartition)) + return t, nil +} -func (broker *MessageBroker) checkPeers() { +func (broker *MessageBroker) checkFilers() { // contact a filer about masters var masters []string diff --git a/weed/messaging/broker/broker_grpc_server_publish.go b/weed/messaging/broker/broker_grpc_server_publish.go index b3a909a6c..61e53b433 100644 --- a/weed/messaging/broker/broker_grpc_server_publish.go +++ b/weed/messaging/broker/broker_grpc_server_publish.go @@ -47,24 +47,11 @@ func (broker *MessageBroker) Publish(stream messaging_pb.SeaweedMessaging_Publis tl := broker.topicLocks.RequestLock(tp, topicConfig, true) defer broker.topicLocks.ReleaseLock(tp, true) - updatesChan := make(chan int32) - - go func() { - for update := range updatesChan { - if err := stream.Send(&messaging_pb.PublishResponse{ - Config: &messaging_pb.PublishResponse_ConfigMessage{ - PartitionCount: update, - }, - }); err != nil { - glog.V(0).Infof("err sending publish response: %v", err) - return - } - } - }() - // process each message for { + // println("recv") in, err := stream.Recv() + // glog.V(0).Infof("recieved %v err: %v", in, err) if err == io.EOF { return nil } @@ -86,5 +73,18 @@ func (broker *MessageBroker) Publish(stream messaging_pb.SeaweedMessaging_Publis tl.logBuffer.AddToBuffer(in.Data.Key, data) + if in.Data.IsClose { + // println("server received closing") + break + } + } + + // send the close ack + // println("server send ack closing") + if err := stream.Send(&messaging_pb.PublishResponse{IsClosed: true}); err != nil { + glog.V(0).Infof("err sending close response: %v", err) + } + return nil + } diff --git a/weed/messaging/broker/broker_grpc_server_subscribe.go b/weed/messaging/broker/broker_grpc_server_subscribe.go index 472a5007b..761129e80 100644 --- a/weed/messaging/broker/broker_grpc_server_subscribe.go +++ b/weed/messaging/broker/broker_grpc_server_subscribe.go @@ -83,11 +83,17 @@ func (broker *MessageBroker) Subscribe(stream messaging_pb.SeaweedMessaging_Subs glog.Errorf("sending %d bytes to %s: %s", len(m.Value), subscriberId, err) return err } + if m.IsClose { + // println("processed EOF") + return io.EOF + } processedTsNs = logEntry.TsNs + messageCount++ return nil } if err := broker.readPersistedLogBuffer(&tp, lastReadTime, eachLogEntryFn); err != nil { + // println("stopping from persisted logs") return err } @@ -95,7 +101,7 @@ func (broker *MessageBroker) Subscribe(stream messaging_pb.SeaweedMessaging_Subs lastReadTime = time.Unix(0, processedTsNs) } - messageCount, err = lock.logBuffer.LoopProcessLogData(lastReadTime, func() bool { + err = lock.logBuffer.LoopProcessLogData(lastReadTime, func() bool { lock.Mutex.Lock() lock.cond.Wait() lock.Mutex.Unlock() @@ -124,7 +130,7 @@ func (broker *MessageBroker) readPersistedLogBuffer(tp *TopicPartition, startTim return nil } } - if !strings.HasSuffix(hourMinuteEntry.Name, partitionSuffix){ + if !strings.HasSuffix(hourMinuteEntry.Name, partitionSuffix) { return nil } // println("partition", tp.Partition, "processing", dayDir, "/", hourMinuteEntry.Name) @@ -133,7 +139,7 @@ func (broker *MessageBroker) readPersistedLogBuffer(tp *TopicPartition, startTim if err := filer2.ReadEachLogEntry(chunkedFileReader, sizeBuf, startTsNs, eachLogEntryFn); err != nil { chunkedFileReader.Close() if err == io.EOF { - return nil + return err } return fmt.Errorf("reading %s/%s: %v", dayDir, hourMinuteEntry.Name, err) } diff --git a/weed/messaging/broker/broker_server.go b/weed/messaging/broker/broker_server.go index 9cad27214..e6ff2cf00 100644 --- a/weed/messaging/broker/broker_server.go +++ b/weed/messaging/broker/broker_server.go @@ -36,7 +36,7 @@ func NewMessageBroker(option *MessageBrokerOption, grpcDialOption grpc.DialOptio messageBroker.topicLocks = NewTopicLocks(messageBroker) - messageBroker.checkPeers() + messageBroker.checkFilers() go messageBroker.keepConnectedToOneFiler() @@ -53,6 +53,24 @@ func (broker *MessageBroker) keepConnectedToOneFiler() { glog.V(0).Infof("%s:%d failed to keep connected to %s: %v", broker.option.Ip, broker.option.Port, filer, err) return err } + + initRequest := &filer_pb.KeepConnectedRequest{ + Name: broker.option.Ip, + GrpcPort: uint32(broker.option.Port), + } + for _, tp := range broker.topicLocks.ListTopicPartitions() { + initRequest.Resources = append(initRequest.Resources, tp.String()) + } + if err := stream.Send(&filer_pb.KeepConnectedRequest{ + Name: broker.option.Ip, + GrpcPort: uint32(broker.option.Port), + }); err != nil { + glog.V(0).Infof("broker %s:%d failed to init at %s: %v", broker.option.Ip, broker.option.Port, filer, err) + return err + } + + // TODO send events of adding/removing topics + glog.V(0).Infof("conntected with filer: %v", filer) for { if err := stream.Send(&filer_pb.KeepConnectedRequest{ @@ -68,12 +86,12 @@ func (broker *MessageBroker) keepConnectedToOneFiler() { return err } // println("received reply") - time.Sleep(11*time.Second) + time.Sleep(11 * time.Second) // println("woke up") } return nil }) - time.Sleep(3*time.Second) + time.Sleep(3 * time.Second) } } diff --git a/weed/messaging/broker/consistent_distribution.go b/weed/messaging/broker/consistent_distribution.go new file mode 100644 index 000000000..dd7d34f86 --- /dev/null +++ b/weed/messaging/broker/consistent_distribution.go @@ -0,0 +1,38 @@ +package broker + +import ( + "github.com/cespare/xxhash" + "github.com/buraksezer/consistent" +) + +type Member string + +func (m Member) String() string { + return string(m) +} + +type hasher struct{} + +func (h hasher) Sum64(data []byte) uint64 { + return xxhash.Sum64(data) +} + +func PickMember(members []string, key []byte) string { + cfg := consistent.Config{ + PartitionCount: 9791, + ReplicationFactor: 2, + Load: 1.25, + Hasher: hasher{}, + } + + cmembers := []consistent.Member{} + for _, m := range members { + cmembers = append(cmembers, Member(m)) + } + + c := consistent.New(cmembers, cfg) + + m := c.LocateKey(key) + + return m.String() +} \ No newline at end of file diff --git a/weed/messaging/broker/consistent_distribution_test.go b/weed/messaging/broker/consistent_distribution_test.go new file mode 100644 index 000000000..192516092 --- /dev/null +++ b/weed/messaging/broker/consistent_distribution_test.go @@ -0,0 +1,32 @@ +package broker + +import ( + "fmt" + "testing" +) + +func TestPickMember(t *testing.T) { + + servers := []string{ + "s1:port", + "s2:port", + "s3:port", + "s5:port", + "s4:port", + } + + total := 1000 + + distribution := make(map[string]int) + for i:=0;i", m) + distribution[m]++ + } + + for member, count := range distribution { + fmt.Printf("member: %s, key count: %d load=%.2f\n", member, count, float64(count*100)/float64(total/len(servers))) + } + +} \ No newline at end of file diff --git a/weed/messaging/broker/topic_lock.go b/weed/messaging/broker/topic_lock.go index f8a5aa171..f3a66a2f5 100644 --- a/weed/messaging/broker/topic_lock.go +++ b/weed/messaging/broker/topic_lock.go @@ -16,6 +16,13 @@ type TopicPartition struct { Topic string Partition int32 } +const ( + TopicPartitionFmt = "%s/%s_%2d" +) +func (tp *TopicPartition) String() string { + return fmt.Sprintf(TopicPartitionFmt, tp.Namespace, tp.Topic, tp.Partition) +} + type TopicLock struct { sync.Mutex cond *sync.Cond @@ -101,3 +108,13 @@ func (tl *TopicLocks) ReleaseLock(partition TopicPartition, isPublisher bool) { delete(tl.locks, partition) } } + +func (tl *TopicLocks) ListTopicPartitions() (tps []TopicPartition) { + tl.Lock() + defer tl.Unlock() + + for k := range tl.locks { + tps = append(tps, k) + } + return +} diff --git a/weed/messaging/msgclient/pub_sub_chan.go b/weed/messaging/msgclient/pub_sub_chan.go index d39e4c658..a11240080 100644 --- a/weed/messaging/msgclient/pub_sub_chan.go +++ b/weed/messaging/msgclient/pub_sub_chan.go @@ -5,12 +5,15 @@ import ( "log" "time" + "google.golang.org/grpc" + "github.com/chrislusf/seaweedfs/weed/messaging/broker" "github.com/chrislusf/seaweedfs/weed/pb/messaging_pb" ) type PubChannel struct { - client messaging_pb.SeaweedMessaging_PublishClient + client messaging_pb.SeaweedMessaging_PublishClient + grpcConnection *grpc.ClientConn } func (mc *MessagingClient) NewPubChannel(chanName string) (*PubChannel, error) { @@ -28,7 +31,8 @@ func (mc *MessagingClient) NewPubChannel(chanName string) (*PubChannel, error) { return nil, err } return &PubChannel{ - client: pc, + client: pc, + grpcConnection: grpcConnection, }, nil } @@ -40,7 +44,24 @@ func (pc *PubChannel) Publish(m []byte) error { }) } func (pc *PubChannel) Close() error { - return pc.client.CloseSend() + + // println("send closing") + if err := pc.client.Send(&messaging_pb.PublishRequest{ + Data: &messaging_pb.Message{ + IsClose: true, + }, + }); err != nil { + log.Printf("err send close: %v", err) + } + // println("receive closing") + if _, err := pc.client.Recv(); err != nil && err != io.EOF { + log.Printf("err receive close: %v", err) + } + // println("close connection") + if err := pc.grpcConnection.Close(); err != nil { + log.Printf("err connection close: %v", err) + } + return nil } type SubChannel struct { @@ -58,7 +79,7 @@ func (mc *MessagingClient) NewSubChannel(chanName string) (*SubChannel, error) { if err != nil { return nil, err } - sc, err := setupSubscriberClient(grpcConnection, "", "chan", chanName, 0, time.Unix(0,0)) + sc, err := setupSubscriberClient(grpcConnection, "", "chan", chanName, 0, time.Unix(0, 0)) if err != nil { return nil, err } @@ -78,13 +99,14 @@ func (mc *MessagingClient) NewSubChannel(chanName string) (*SubChannel, error) { log.Printf("fail to receive from netchan %s: %v", chanName, subErr) return } - if resp.IsClose { + if resp.Data.IsClose { + t.stream.Send(&messaging_pb.SubscriberMessage{ + IsClose: true, + }) close(t.ch) return } - if resp.Data != nil { - t.ch <- resp.Data.Value - } + t.ch <- resp.Data.Value } }() diff --git a/weed/messaging/msgclient/publisher.go b/weed/messaging/msgclient/publisher.go index b0459494b..08f1d278a 100644 --- a/weed/messaging/msgclient/publisher.go +++ b/weed/messaging/msgclient/publisher.go @@ -4,9 +4,9 @@ import ( "context" "github.com/OneOfOne/xxhash" + "google.golang.org/grpc" "github.com/chrislusf/seaweedfs/weed/messaging/broker" - "github.com/chrislusf/seaweedfs/weed/pb" "github.com/chrislusf/seaweedfs/weed/pb/messaging_pb" ) @@ -16,7 +16,7 @@ type Publisher struct { messageCount uint64 publisherId string } - +/* func (mc *MessagingClient) NewPublisher(publisherId, namespace, topic string) (*Publisher, error) { // read topic configuration topicConfiguration := &messaging_pb.TopicConfiguration{ @@ -24,7 +24,11 @@ func (mc *MessagingClient) NewPublisher(publisherId, namespace, topic string) (* } publishClients := make([]messaging_pb.SeaweedMessaging_PublishClient, topicConfiguration.PartitionCount) for i := 0; i < int(topicConfiguration.PartitionCount); i++ { - client, err := mc.setupPublisherClient(namespace, topic, int32(i)) + client, err := setupPublisherClient(broker.TopicPartition{ + Namespace: namespace, + Topic: topic, + Partition: int32(i), + }) if err != nil { return nil, err } @@ -35,6 +39,7 @@ func (mc *MessagingClient) NewPublisher(publisherId, namespace, topic string) (* topicConfiguration: topicConfiguration, }, nil } +*/ func setupPublisherClient(grpcConnection *grpc.ClientConn, tp broker.TopicPartition) (messaging_pb.SeaweedMessaging_PublishClient, error) { diff --git a/weed/messaging/msgclient/subscriber.go b/weed/messaging/msgclient/subscriber.go index 27fa35a5b..d3066d6ef 100644 --- a/weed/messaging/msgclient/subscriber.go +++ b/weed/messaging/msgclient/subscriber.go @@ -5,6 +5,7 @@ import ( "io" "time" + "google.golang.org/grpc" "github.com/chrislusf/seaweedfs/weed/pb/messaging_pb" ) @@ -13,6 +14,7 @@ type Subscriber struct { subscriberId string } +/* func (mc *MessagingClient) NewSubscriber(subscriberId, namespace, topic string, startTime time.Time) (*Subscriber, error) { // read topic configuration topicConfiguration := &messaging_pb.TopicConfiguration{ @@ -36,9 +38,9 @@ func (mc *MessagingClient) NewSubscriber(subscriberId, namespace, topic string, func (mc *MessagingClient) setupSubscriberClient(subscriberId, namespace, topic string, partition int32, startTime time.Time) (messaging_pb.SeaweedMessaging_SubscribeClient, error) { - stream, newBroker, err := mc.initSubscriberClient(subscriberId, namespace, topic, partition, startTime) + stream, err := setupSubscriberClient(subscriberId, namespace, topic, partition, startTime) if err != nil { - return client, err + return stream, err } if newBroker != nil { @@ -47,6 +49,7 @@ func (mc *MessagingClient) setupSubscriberClient(subscriberId, namespace, topic return stream, nil } +*/ func setupSubscriberClient(grpcConnection *grpc.ClientConn, subscriberId string, namespace string, topic string, partition int32, startTime time.Time) (stream messaging_pb.SeaweedMessaging_SubscribeClient, err error) { stream, err = messaging_pb.NewSeaweedMessagingClient(grpcConnection).Subscribe(context.Background()) @@ -70,13 +73,10 @@ func setupSubscriberClient(grpcConnection *grpc.ClientConn, subscriberId string, } // process init response - initResponse, err := stream.Recv() + _, err = stream.Recv() if err != nil { return } - if initResponse.Redirect != nil { - // TODO follow redirection - } return stream, nil } diff --git a/weed/pb/filer.proto b/weed/pb/filer.proto index 3b3b78bbb..1fc8ef63d 100644 --- a/weed/pb/filer.proto +++ b/weed/pb/filer.proto @@ -50,6 +50,10 @@ service SeaweedFiler { rpc KeepConnected (stream KeepConnectedRequest) returns (stream KeepConnectedResponse) { } + + rpc LocateBroker (LocateBrokerRequest) returns (LocateBrokerResponse) { + } + } ////////////////////////////////////////////////// @@ -267,6 +271,21 @@ message LogEntry { message KeepConnectedRequest { string name = 1; uint32 grpc_port = 2; + repeated string resources = 3; } message KeepConnectedResponse { } + +message LocateBrokerRequest { + string resource = 1; +} +message LocateBrokerResponse { + bool found = 1; + // if found, send the exact address + // if not found, send the full list of existing brokers + message Resource { + string grpc_addresses = 1; + int32 resource_count = 2; + } + repeated Resource resources = 2; +} diff --git a/weed/pb/filer_pb/filer.pb.go b/weed/pb/filer_pb/filer.pb.go index f1320273f..f5b62e377 100644 --- a/weed/pb/filer_pb/filer.pb.go +++ b/weed/pb/filer_pb/filer.pb.go @@ -46,6 +46,8 @@ It has these top-level messages: LogEntry KeepConnectedRequest KeepConnectedResponse + LocateBrokerRequest + LocateBrokerResponse */ package filer_pb @@ -1231,8 +1233,9 @@ func (m *LogEntry) GetData() []byte { } type KeepConnectedRequest struct { - Name string `protobuf:"bytes,1,opt,name=name" json:"name,omitempty"` - GrpcPort uint32 `protobuf:"varint,2,opt,name=grpc_port,json=grpcPort" json:"grpc_port,omitempty"` + Name string `protobuf:"bytes,1,opt,name=name" json:"name,omitempty"` + GrpcPort uint32 `protobuf:"varint,2,opt,name=grpc_port,json=grpcPort" json:"grpc_port,omitempty"` + Resources []string `protobuf:"bytes,3,rep,name=resources" json:"resources,omitempty"` } func (m *KeepConnectedRequest) Reset() { *m = KeepConnectedRequest{} } @@ -1254,6 +1257,13 @@ func (m *KeepConnectedRequest) GetGrpcPort() uint32 { return 0 } +func (m *KeepConnectedRequest) GetResources() []string { + if m != nil { + return m.Resources + } + return nil +} + type KeepConnectedResponse struct { } @@ -1262,6 +1272,74 @@ func (m *KeepConnectedResponse) String() string { return proto.Compac func (*KeepConnectedResponse) ProtoMessage() {} func (*KeepConnectedResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{36} } +type LocateBrokerRequest struct { + Resource string `protobuf:"bytes,1,opt,name=resource" json:"resource,omitempty"` +} + +func (m *LocateBrokerRequest) Reset() { *m = LocateBrokerRequest{} } +func (m *LocateBrokerRequest) String() string { return proto.CompactTextString(m) } +func (*LocateBrokerRequest) ProtoMessage() {} +func (*LocateBrokerRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{37} } + +func (m *LocateBrokerRequest) GetResource() string { + if m != nil { + return m.Resource + } + return "" +} + +type LocateBrokerResponse struct { + Found bool `protobuf:"varint,1,opt,name=found" json:"found,omitempty"` + Resources []*LocateBrokerResponse_Resource `protobuf:"bytes,2,rep,name=resources" json:"resources,omitempty"` +} + +func (m *LocateBrokerResponse) Reset() { *m = LocateBrokerResponse{} } +func (m *LocateBrokerResponse) String() string { return proto.CompactTextString(m) } +func (*LocateBrokerResponse) ProtoMessage() {} +func (*LocateBrokerResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{38} } + +func (m *LocateBrokerResponse) GetFound() bool { + if m != nil { + return m.Found + } + return false +} + +func (m *LocateBrokerResponse) GetResources() []*LocateBrokerResponse_Resource { + if m != nil { + return m.Resources + } + return nil +} + +// if found, send the exact address +// if not found, send the full list of existing brokers +type LocateBrokerResponse_Resource struct { + GrpcAddresses string `protobuf:"bytes,1,opt,name=grpc_addresses,json=grpcAddresses" json:"grpc_addresses,omitempty"` + ResourceCount int32 `protobuf:"varint,2,opt,name=resource_count,json=resourceCount" json:"resource_count,omitempty"` +} + +func (m *LocateBrokerResponse_Resource) Reset() { *m = LocateBrokerResponse_Resource{} } +func (m *LocateBrokerResponse_Resource) String() string { return proto.CompactTextString(m) } +func (*LocateBrokerResponse_Resource) ProtoMessage() {} +func (*LocateBrokerResponse_Resource) Descriptor() ([]byte, []int) { + return fileDescriptor0, []int{38, 0} +} + +func (m *LocateBrokerResponse_Resource) GetGrpcAddresses() string { + if m != nil { + return m.GrpcAddresses + } + return "" +} + +func (m *LocateBrokerResponse_Resource) GetResourceCount() int32 { + if m != nil { + return m.ResourceCount + } + return 0 +} + func init() { proto.RegisterType((*LookupDirectoryEntryRequest)(nil), "filer_pb.LookupDirectoryEntryRequest") proto.RegisterType((*LookupDirectoryEntryResponse)(nil), "filer_pb.LookupDirectoryEntryResponse") @@ -1300,6 +1378,9 @@ func init() { proto.RegisterType((*LogEntry)(nil), "filer_pb.LogEntry") proto.RegisterType((*KeepConnectedRequest)(nil), "filer_pb.KeepConnectedRequest") proto.RegisterType((*KeepConnectedResponse)(nil), "filer_pb.KeepConnectedResponse") + proto.RegisterType((*LocateBrokerRequest)(nil), "filer_pb.LocateBrokerRequest") + proto.RegisterType((*LocateBrokerResponse)(nil), "filer_pb.LocateBrokerResponse") + proto.RegisterType((*LocateBrokerResponse_Resource)(nil), "filer_pb.LocateBrokerResponse.Resource") } // Reference imports to suppress errors if they are not otherwise used. @@ -1327,6 +1408,7 @@ type SeaweedFilerClient interface { GetFilerConfiguration(ctx context.Context, in *GetFilerConfigurationRequest, opts ...grpc.CallOption) (*GetFilerConfigurationResponse, error) SubscribeMetadata(ctx context.Context, in *SubscribeMetadataRequest, opts ...grpc.CallOption) (SeaweedFiler_SubscribeMetadataClient, error) KeepConnected(ctx context.Context, opts ...grpc.CallOption) (SeaweedFiler_KeepConnectedClient, error) + LocateBroker(ctx context.Context, in *LocateBrokerRequest, opts ...grpc.CallOption) (*LocateBrokerResponse, error) } type seaweedFilerClient struct { @@ -1531,6 +1613,15 @@ func (x *seaweedFilerKeepConnectedClient) Recv() (*KeepConnectedResponse, error) return m, nil } +func (c *seaweedFilerClient) LocateBroker(ctx context.Context, in *LocateBrokerRequest, opts ...grpc.CallOption) (*LocateBrokerResponse, error) { + out := new(LocateBrokerResponse) + err := grpc.Invoke(ctx, "/filer_pb.SeaweedFiler/LocateBroker", in, out, c.cc, opts...) + if err != nil { + return nil, err + } + return out, nil +} + // Server API for SeaweedFiler service type SeaweedFilerServer interface { @@ -1548,6 +1639,7 @@ type SeaweedFilerServer interface { GetFilerConfiguration(context.Context, *GetFilerConfigurationRequest) (*GetFilerConfigurationResponse, error) SubscribeMetadata(*SubscribeMetadataRequest, SeaweedFiler_SubscribeMetadataServer) error KeepConnected(SeaweedFiler_KeepConnectedServer) error + LocateBroker(context.Context, *LocateBrokerRequest) (*LocateBrokerResponse, error) } func RegisterSeaweedFilerServer(s *grpc.Server, srv SeaweedFilerServer) { @@ -1820,6 +1912,24 @@ func (x *seaweedFilerKeepConnectedServer) Recv() (*KeepConnectedRequest, error) return m, nil } +func _SeaweedFiler_LocateBroker_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(LocateBrokerRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(SeaweedFilerServer).LocateBroker(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/filer_pb.SeaweedFiler/LocateBroker", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(SeaweedFilerServer).LocateBroker(ctx, req.(*LocateBrokerRequest)) + } + return interceptor(ctx, in, info, handler) +} + var _SeaweedFiler_serviceDesc = grpc.ServiceDesc{ ServiceName: "filer_pb.SeaweedFiler", HandlerType: (*SeaweedFilerServer)(nil), @@ -1868,6 +1978,10 @@ var _SeaweedFiler_serviceDesc = grpc.ServiceDesc{ MethodName: "GetFilerConfiguration", Handler: _SeaweedFiler_GetFilerConfiguration_Handler, }, + { + MethodName: "LocateBroker", + Handler: _SeaweedFiler_LocateBroker_Handler, + }, }, Streams: []grpc.StreamDesc{ { @@ -1893,132 +2007,139 @@ var _SeaweedFiler_serviceDesc = grpc.ServiceDesc{ func init() { proto.RegisterFile("filer.proto", fileDescriptor0) } var fileDescriptor0 = []byte{ - // 2020 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xb4, 0x58, 0xcd, 0x6e, 0xdc, 0xc8, - 0x11, 0x16, 0x67, 0x34, 0xa3, 0x61, 0xcd, 0x8c, 0x57, 0x6a, 0x49, 0xf6, 0x68, 0xf4, 0x63, 0x2d, - 0x1d, 0x6f, 0x14, 0xd8, 0x50, 0x0c, 0x65, 0x03, 0xec, 0x66, 0x93, 0x83, 0x2d, 0xcb, 0x8e, 0x63, - 0x5b, 0x2b, 0x50, 0xf6, 0x22, 0x41, 0x80, 0x30, 0x14, 0xd9, 0x1a, 0x75, 0xc4, 0x21, 0x99, 0xee, - 0xa6, 0x7e, 0xf6, 0xb4, 0xcf, 0x11, 0x20, 0x6f, 0x91, 0x63, 0x90, 0x4b, 0x10, 0x20, 0x40, 0xce, - 0x79, 0x81, 0x3c, 0x49, 0xd0, 0xd5, 0x24, 0xa7, 0x39, 0x3f, 0xd2, 0x1a, 0x0b, 0xdf, 0xba, 0xab, - 0xaa, 0xab, 0xab, 0xeb, 0xe7, 0xab, 0x22, 0xa1, 0x7d, 0xca, 0x22, 0xca, 0x77, 0x53, 0x9e, 0xc8, - 0x84, 0xb4, 0x70, 0xe3, 0xa5, 0x27, 0xce, 0xd7, 0xb0, 0xfe, 0x26, 0x49, 0xce, 0xb3, 0xf4, 0x39, - 0xe3, 0x34, 0x90, 0x09, 0xbf, 0x3e, 0x88, 0x25, 0xbf, 0x76, 0xe9, 0x9f, 0x33, 0x2a, 0x24, 0xd9, - 0x00, 0x3b, 0x2c, 0x18, 0x3d, 0x6b, 0xdb, 0xda, 0xb1, 0xdd, 0x11, 0x81, 0x10, 0x98, 0x8f, 0xfd, - 0x21, 0xed, 0xd5, 0x90, 0x81, 0x6b, 0xe7, 0x00, 0x36, 0xa6, 0x2b, 0x14, 0x69, 0x12, 0x0b, 0x4a, - 0x1e, 0x42, 0x83, 0x2a, 0x02, 0x6a, 0x6b, 0xef, 0x7d, 0xb2, 0x5b, 0x98, 0xb2, 0xab, 0xe5, 0x34, - 0xd7, 0xf9, 0x87, 0x05, 0xe4, 0x0d, 0x13, 0x52, 0x11, 0x19, 0x15, 0xdf, 0xcf, 0x9e, 0xbb, 0xd0, - 0x4c, 0x39, 0x3d, 0x65, 0x57, 0xb9, 0x45, 0xf9, 0x8e, 0x3c, 0x86, 0x25, 0x21, 0x7d, 0x2e, 0x5f, - 0xf0, 0x64, 0xf8, 0x82, 0x45, 0xf4, 0x50, 0x19, 0x5d, 0x47, 0x91, 0x49, 0x06, 0xd9, 0x05, 0xc2, - 0xe2, 0x20, 0xca, 0x04, 0xbb, 0xa0, 0xc7, 0x05, 0xb7, 0x37, 0xbf, 0x6d, 0xed, 0xb4, 0xdc, 0x29, - 0x1c, 0xb2, 0x02, 0x8d, 0x88, 0x0d, 0x99, 0xec, 0x35, 0xb6, 0xad, 0x9d, 0xae, 0xab, 0x37, 0xce, - 0x2f, 0x61, 0xb9, 0x62, 0xff, 0x87, 0x3d, 0xff, 0xaf, 0x35, 0x68, 0x20, 0xa1, 0xf4, 0xb1, 0x35, - 0xf2, 0x31, 0xf9, 0x14, 0x3a, 0x4c, 0x78, 0x23, 0x47, 0xd4, 0xd0, 0xb6, 0x36, 0x13, 0xa5, 0xcf, - 0xc9, 0x23, 0x68, 0x06, 0x67, 0x59, 0x7c, 0x2e, 0x7a, 0xf5, 0xed, 0xfa, 0x4e, 0x7b, 0x6f, 0x79, - 0x74, 0x91, 0x7a, 0xe8, 0xbe, 0xe2, 0xb9, 0xb9, 0x08, 0xf9, 0x02, 0xc0, 0x97, 0x92, 0xb3, 0x93, - 0x4c, 0x52, 0x81, 0x2f, 0x6d, 0xef, 0xf5, 0x8c, 0x03, 0x99, 0xa0, 0x4f, 0x4b, 0xbe, 0x6b, 0xc8, - 0x92, 0x2f, 0xa1, 0x45, 0xaf, 0x24, 0x8d, 0x43, 0x1a, 0xf6, 0x1a, 0x78, 0xd1, 0xe6, 0xd8, 0x8b, - 0x76, 0x0f, 0x72, 0xbe, 0x7e, 0x5f, 0x29, 0xde, 0xff, 0x0a, 0xba, 0x15, 0x16, 0x59, 0x84, 0xfa, - 0x39, 0x2d, 0xa2, 0xaa, 0x96, 0xca, 0xb3, 0x17, 0x7e, 0x94, 0xe9, 0x04, 0xeb, 0xb8, 0x7a, 0xf3, - 0x8b, 0xda, 0x17, 0x96, 0xf3, 0x1c, 0xec, 0x17, 0x59, 0x14, 0x95, 0x07, 0x43, 0xc6, 0x8b, 0x83, - 0x21, 0xe3, 0x23, 0x2f, 0xd7, 0x6e, 0xf4, 0xf2, 0xdf, 0x2d, 0x58, 0x3a, 0xb8, 0xa0, 0xb1, 0x3c, - 0x4c, 0x24, 0x3b, 0x65, 0x81, 0x2f, 0x59, 0x12, 0x93, 0xc7, 0x60, 0x27, 0x51, 0xe8, 0xdd, 0x18, - 0xa6, 0x56, 0x12, 0xe5, 0x56, 0x3f, 0x06, 0x3b, 0xa6, 0x97, 0xde, 0x8d, 0xd7, 0xb5, 0x62, 0x7a, - 0xa9, 0xa5, 0x1f, 0x40, 0x37, 0xa4, 0x11, 0x95, 0xd4, 0x2b, 0xa3, 0xa3, 0x42, 0xd7, 0xd1, 0xc4, - 0x7d, 0x1d, 0x8e, 0xcf, 0xe0, 0x13, 0xa5, 0x32, 0xf5, 0x39, 0x8d, 0xa5, 0x97, 0xfa, 0xf2, 0x0c, - 0x63, 0x62, 0xbb, 0xdd, 0x98, 0x5e, 0x1e, 0x21, 0xf5, 0xc8, 0x97, 0x67, 0xce, 0xdf, 0x6a, 0x60, - 0x97, 0xc1, 0x24, 0xf7, 0x60, 0x41, 0x5d, 0xeb, 0xb1, 0x30, 0xf7, 0x44, 0x53, 0x6d, 0x5f, 0x85, - 0xaa, 0x2a, 0x92, 0xd3, 0x53, 0x41, 0x25, 0x9a, 0x57, 0x77, 0xf3, 0x9d, 0xca, 0x2c, 0xc1, 0xbe, - 0xd5, 0x85, 0x30, 0xef, 0xe2, 0x5a, 0x79, 0x7c, 0x28, 0xd9, 0x90, 0xe2, 0x85, 0x75, 0x57, 0x6f, - 0xc8, 0x32, 0x34, 0xa8, 0x27, 0xfd, 0x01, 0x66, 0xb8, 0xed, 0xce, 0xd3, 0x77, 0xfe, 0x80, 0xfc, - 0x08, 0xee, 0x88, 0x24, 0xe3, 0x01, 0xf5, 0x8a, 0x6b, 0x9b, 0xc8, 0xed, 0x68, 0xea, 0x0b, 0x7d, - 0xb9, 0x03, 0xf5, 0x53, 0x16, 0xf6, 0x16, 0xd0, 0x31, 0x8b, 0xd5, 0x24, 0x7c, 0x15, 0xba, 0x8a, - 0x49, 0x7e, 0x0a, 0x50, 0x6a, 0x0a, 0x7b, 0xad, 0x19, 0xa2, 0x76, 0xa1, 0x37, 0x24, 0x9b, 0x00, - 0x01, 0x4b, 0xcf, 0x28, 0xf7, 0x54, 0xc2, 0xd8, 0x98, 0x1c, 0xb6, 0xa6, 0xbc, 0xa6, 0xd7, 0x8a, - 0xcd, 0x84, 0x37, 0xf8, 0x96, 0xa5, 0x29, 0x0d, 0x7b, 0x80, 0x1e, 0xb6, 0x99, 0x78, 0xa9, 0x09, - 0xce, 0x6f, 0xa1, 0x99, 0x1b, 0xb7, 0x0e, 0xf6, 0x45, 0x12, 0x65, 0xc3, 0xd2, 0x69, 0x5d, 0xb7, - 0xa5, 0x09, 0xaf, 0x42, 0xb2, 0x06, 0x88, 0x92, 0x78, 0x45, 0x0d, 0x5d, 0x84, 0xfe, 0x55, 0x17, - 0xdc, 0x85, 0x66, 0x90, 0x24, 0xe7, 0x4c, 0xfb, 0x6e, 0xc1, 0xcd, 0x77, 0xce, 0x77, 0x75, 0xb8, - 0x53, 0x2d, 0x16, 0x75, 0x05, 0x6a, 0x41, 0x4f, 0x5b, 0xa8, 0x06, 0xd5, 0x1e, 0x57, 0xbc, 0x5d, - 0x33, 0xbd, 0x5d, 0x1c, 0x19, 0x26, 0xa1, 0xbe, 0xa0, 0xab, 0x8f, 0xbc, 0x4d, 0x42, 0xaa, 0x72, - 0x3d, 0x63, 0x21, 0x86, 0xa7, 0xeb, 0xaa, 0xa5, 0xa2, 0x0c, 0x58, 0x98, 0x83, 0x8f, 0x5a, 0xa2, - 0x79, 0x1c, 0xf5, 0x36, 0x75, 0xc0, 0xf5, 0x4e, 0x05, 0x7c, 0xa8, 0xa8, 0x0b, 0x3a, 0x8a, 0x6a, - 0x4d, 0xb6, 0xa1, 0xcd, 0x69, 0x1a, 0xe5, 0xb9, 0x8f, 0xce, 0xb7, 0x5d, 0x93, 0x44, 0xb6, 0x00, - 0x82, 0x24, 0x8a, 0x68, 0x80, 0x02, 0x36, 0x0a, 0x18, 0x14, 0x95, 0x77, 0x52, 0x46, 0x9e, 0xa0, - 0x01, 0xba, 0xba, 0xe1, 0x36, 0xa5, 0x8c, 0x8e, 0x69, 0xa0, 0xde, 0x91, 0x09, 0xca, 0x3d, 0x84, - 0xaf, 0x36, 0x9e, 0x6b, 0x29, 0x02, 0x82, 0xec, 0x26, 0xc0, 0x80, 0x27, 0x59, 0xaa, 0xb9, 0x9d, - 0xed, 0xba, 0x42, 0x72, 0xa4, 0x20, 0xfb, 0x21, 0xdc, 0x11, 0xd7, 0xc3, 0x88, 0xc5, 0xe7, 0x9e, - 0xf4, 0xf9, 0x80, 0xca, 0x5e, 0x57, 0x57, 0x40, 0x4e, 0x7d, 0x87, 0x44, 0xf5, 0xf6, 0x61, 0xf8, - 0xf3, 0xde, 0x1d, 0xcc, 0x00, 0xb5, 0x74, 0x52, 0x20, 0xfb, 0x9c, 0xfa, 0x92, 0x7e, 0x40, 0x1b, - 0xfb, 0x7e, 0x68, 0x41, 0x56, 0xa1, 0x99, 0x78, 0xf4, 0x2a, 0x88, 0xf2, 0xa2, 0x6d, 0x24, 0x07, - 0x57, 0x41, 0xe4, 0x3c, 0x82, 0xe5, 0xca, 0x8d, 0x39, 0xd0, 0xaf, 0x40, 0x83, 0x72, 0x9e, 0x14, - 0xb0, 0xa4, 0x37, 0xce, 0xef, 0x80, 0xbc, 0x4f, 0xc3, 0x8f, 0x61, 0x9e, 0xb3, 0x0a, 0xcb, 0x15, - 0xd5, 0xda, 0x0e, 0xe7, 0x3b, 0x0b, 0x56, 0x9e, 0xa6, 0x29, 0x8d, 0xc3, 0x77, 0xc9, 0x07, 0x5c, - 0xba, 0x09, 0x80, 0x6a, 0x3d, 0xa3, 0xc1, 0xdb, 0x48, 0xc1, 0xf8, 0x7c, 0x48, 0x7b, 0x71, 0xee, - 0xc1, 0xea, 0x98, 0x05, 0xb9, 0x6d, 0xff, 0xb2, 0x80, 0x3c, 0x47, 0xe4, 0xfb, 0x61, 0x43, 0x87, - 0xc2, 0x22, 0xd5, 0x10, 0x35, 0xb2, 0x86, 0xbe, 0xf4, 0xf3, 0x76, 0xdd, 0x61, 0x42, 0xeb, 0x7f, - 0xee, 0x4b, 0x3f, 0x6f, 0x9b, 0x9c, 0x06, 0x19, 0x57, 0x1d, 0x1c, 0x4b, 0x06, 0xdb, 0xa6, 0x5b, - 0x90, 0xc8, 0xe7, 0x70, 0x97, 0x0d, 0xe2, 0x84, 0xd3, 0x91, 0x98, 0xa7, 0xc3, 0xd8, 0x44, 0xe1, - 0x15, 0xcd, 0x2d, 0x0f, 0x1c, 0x60, 0x54, 0x1f, 0xc1, 0x72, 0xe5, 0x19, 0x37, 0xa6, 0xc0, 0x5f, - 0x2c, 0xe8, 0x3d, 0x95, 0xc9, 0x90, 0x05, 0x2e, 0x55, 0xc6, 0x57, 0x9e, 0xfe, 0x00, 0xba, 0xaa, - 0xf7, 0x8c, 0x3f, 0xbf, 0x93, 0x44, 0xe1, 0xa8, 0xb7, 0xaf, 0x81, 0x6a, 0x3f, 0x66, 0x64, 0x16, - 0x92, 0x28, 0xc4, 0xb8, 0x3c, 0x00, 0xd5, 0x23, 0x8c, 0xf3, 0x7a, 0xca, 0xe9, 0xc4, 0xf4, 0xb2, - 0x72, 0x5e, 0x09, 0xe1, 0x79, 0xdd, 0x58, 0x16, 0x62, 0x7a, 0xa9, 0xce, 0x3b, 0xeb, 0xb0, 0x36, - 0xc5, 0xb6, 0x3c, 0x5c, 0xff, 0xb6, 0x60, 0xf9, 0xa9, 0x10, 0x6c, 0x10, 0x7f, 0x83, 0x20, 0x59, - 0x18, 0xbd, 0x02, 0x8d, 0x20, 0xc9, 0x62, 0x89, 0xc6, 0x36, 0x5c, 0xbd, 0x19, 0xc3, 0x8d, 0xda, - 0x04, 0x6e, 0x8c, 0x21, 0x4f, 0x7d, 0x12, 0x79, 0x0c, 0x64, 0x99, 0xaf, 0x20, 0xcb, 0x7d, 0x68, - 0xab, 0x20, 0x7b, 0x01, 0x8d, 0x25, 0xe5, 0x79, 0x57, 0x02, 0x45, 0xda, 0x47, 0x8a, 0x12, 0x30, - 0xbb, 0xa7, 0x6e, 0x4c, 0x90, 0x8e, 0x5a, 0xe7, 0xff, 0x54, 0x55, 0x54, 0x9e, 0x92, 0xc7, 0x6c, - 0x66, 0x17, 0x55, 0xc0, 0xcb, 0xa3, 0xfc, 0x1d, 0x6a, 0xa9, 0x4a, 0x24, 0xcd, 0x4e, 0x22, 0x16, - 0x78, 0x8a, 0xa1, 0xed, 0xb7, 0x35, 0xe5, 0x3d, 0x8f, 0x46, 0x5e, 0x99, 0x37, 0xbd, 0x42, 0x60, - 0xde, 0xcf, 0xe4, 0x59, 0xd1, 0x49, 0xd5, 0x7a, 0xcc, 0x53, 0xcd, 0xdb, 0x3c, 0xb5, 0x30, 0xe9, - 0xa9, 0x32, 0xd3, 0x5a, 0x66, 0xa6, 0x7d, 0x0e, 0xcb, 0x7a, 0x14, 0xaf, 0x86, 0x6b, 0x13, 0xa0, - 0xec, 0x7a, 0xa2, 0x67, 0x69, 0xe8, 0x2d, 0xda, 0x9e, 0x70, 0x7e, 0x05, 0xf6, 0x9b, 0x44, 0xeb, - 0x15, 0xe4, 0x09, 0xd8, 0x51, 0xb1, 0x41, 0xd1, 0xf6, 0x1e, 0x19, 0x95, 0x7a, 0x21, 0xe7, 0x8e, - 0x84, 0x9c, 0xaf, 0xa0, 0x55, 0x90, 0x0b, 0x9f, 0x59, 0xb3, 0x7c, 0x56, 0x1b, 0xf3, 0x99, 0xf3, - 0x4f, 0x0b, 0x56, 0xaa, 0x26, 0xe7, 0x61, 0x79, 0x0f, 0xdd, 0xf2, 0x0a, 0x6f, 0xe8, 0xa7, 0xb9, - 0x2d, 0x4f, 0x4c, 0x5b, 0x26, 0x8f, 0x95, 0x06, 0x8a, 0xb7, 0x7e, 0xaa, 0x73, 0xb9, 0x13, 0x19, - 0xa4, 0xfe, 0x3b, 0x58, 0x9a, 0x10, 0x99, 0x32, 0x87, 0xfe, 0xc4, 0x9c, 0x43, 0x2b, 0x60, 0x57, - 0x9e, 0x36, 0x87, 0xd3, 0x2f, 0xe1, 0x9e, 0x86, 0x83, 0xfd, 0x32, 0x86, 0x85, 0xef, 0xab, 0xa1, - 0xb6, 0xc6, 0x43, 0xed, 0xf4, 0xa1, 0x37, 0x79, 0x34, 0x2f, 0xbf, 0x01, 0x2c, 0x1d, 0x4b, 0x5f, - 0x32, 0x21, 0x59, 0x50, 0x7e, 0x10, 0x8d, 0xe5, 0x86, 0x75, 0x5b, 0xff, 0x9e, 0xac, 0xc3, 0x45, - 0xa8, 0x4b, 0x59, 0xe4, 0xaf, 0x5a, 0xaa, 0x28, 0x10, 0xf3, 0xa6, 0x3c, 0x06, 0x1f, 0xe1, 0x2a, - 0x95, 0x0f, 0x32, 0x91, 0x7e, 0xa4, 0xe7, 0xa3, 0x79, 0x9c, 0x8f, 0x6c, 0xa4, 0xe0, 0x80, 0xa4, - 0x47, 0x88, 0x50, 0x73, 0x1b, 0x7a, 0x7a, 0x52, 0x04, 0x64, 0x6e, 0x02, 0x60, 0xa9, 0xea, 0x2a, - 0x6b, 0xea, 0xb3, 0x8a, 0xb2, 0xaf, 0x08, 0xce, 0x16, 0x6c, 0xbc, 0xa4, 0x52, 0x75, 0x23, 0xbe, - 0x9f, 0xc4, 0xa7, 0x6c, 0x90, 0x71, 0xdf, 0x08, 0x85, 0xf3, 0x1f, 0x0b, 0x36, 0x67, 0x08, 0xe4, - 0x0f, 0xee, 0xc1, 0xc2, 0xd0, 0x17, 0x92, 0xf2, 0xa2, 0x4a, 0x8a, 0xed, 0xb8, 0x2b, 0x6a, 0xb7, - 0xb9, 0xa2, 0x3e, 0xe1, 0x8a, 0x55, 0x68, 0x0e, 0xfd, 0x2b, 0x6f, 0x78, 0x92, 0x8f, 0x72, 0x8d, - 0xa1, 0x7f, 0xf5, 0xf6, 0x04, 0x91, 0x8d, 0x71, 0xef, 0x24, 0x0b, 0xce, 0xa9, 0x14, 0x25, 0xb2, - 0x31, 0xfe, 0x4c, 0x53, 0x70, 0xb6, 0xc3, 0x41, 0x17, 0x61, 0xa0, 0xe5, 0xe6, 0x3b, 0xe7, 0x12, - 0x7a, 0xc7, 0xd9, 0x89, 0x08, 0x38, 0x3b, 0xa1, 0x6f, 0xa9, 0xf4, 0x15, 0x18, 0x16, 0x39, 0x72, - 0x1f, 0xda, 0x41, 0xc4, 0x14, 0x1a, 0x1a, 0x5f, 0x92, 0xa0, 0x49, 0xd8, 0x35, 0x10, 0x2e, 0xe5, - 0x99, 0x57, 0xf9, 0x78, 0x06, 0x45, 0x3a, 0xd2, 0x1f, 0xd0, 0x6b, 0xd0, 0x12, 0x2c, 0x0e, 0xa8, - 0x17, 0xeb, 0x2f, 0x96, 0xba, 0xbb, 0x80, 0xfb, 0x43, 0xa1, 0xda, 0xd9, 0xda, 0x94, 0x9b, 0x73, - 0x17, 0xde, 0xdc, 0xca, 0x7f, 0x03, 0x84, 0x5e, 0xa0, 0x5d, 0xc6, 0xf7, 0x57, 0x5e, 0x64, 0xeb, - 0xc6, 0x98, 0x33, 0xfe, 0x89, 0xe6, 0x2e, 0xd1, 0x89, 0xaf, 0xb6, 0x65, 0x68, 0x48, 0x31, 0xb2, - 0x6f, 0x5e, 0x8a, 0x43, 0xe1, 0xf8, 0x0a, 0x8c, 0x06, 0xba, 0xac, 0x4b, 0x01, 0x6b, 0x24, 0x40, - 0x1e, 0x03, 0x49, 0x7d, 0x2e, 0x99, 0x52, 0xa1, 0x26, 0x7d, 0xef, 0xcc, 0x17, 0x67, 0x68, 0x41, - 0xc3, 0x5d, 0x2c, 0x39, 0xaf, 0xe9, 0xf5, 0xaf, 0x7d, 0x71, 0xa6, 0xc0, 0x1b, 0x87, 0x8b, 0x3a, - 0xce, 0x9b, 0xb8, 0x76, 0x5e, 0xc2, 0xca, 0x6b, 0x4a, 0xd3, 0xfd, 0x24, 0x8e, 0x69, 0x20, 0x69, - 0x58, 0x38, 0x7d, 0xda, 0x77, 0xfb, 0x3a, 0xd8, 0x03, 0x9e, 0x06, 0x5e, 0x9a, 0x70, 0xfd, 0x31, - 0xd6, 0x75, 0x5b, 0x8a, 0x70, 0x94, 0x70, 0xa9, 0xa6, 0xa4, 0x31, 0x45, 0xda, 0x87, 0x7b, 0xff, - 0xb5, 0xa1, 0x73, 0x4c, 0xfd, 0x4b, 0x4a, 0x43, 0x4c, 0x56, 0x32, 0x28, 0x40, 0xb2, 0xfa, 0x8b, - 0x85, 0x3c, 0x1c, 0x47, 0xc3, 0xa9, 0xff, 0x74, 0xfa, 0x9f, 0xdd, 0x26, 0x96, 0xe3, 0xcd, 0x1c, - 0x39, 0x84, 0xb6, 0xf1, 0x0f, 0x83, 0x6c, 0x18, 0x07, 0x27, 0x7e, 0xcd, 0xf4, 0x37, 0x67, 0x70, - 0x0b, 0x6d, 0x4f, 0x2c, 0xf2, 0x06, 0xda, 0xc6, 0xa8, 0x6c, 0xea, 0x9b, 0x9c, 0xd9, 0x4d, 0x7d, - 0x53, 0xe6, 0x6b, 0x67, 0x4e, 0x69, 0x33, 0x06, 0x5e, 0x53, 0xdb, 0xe4, 0x88, 0x6d, 0x6a, 0x9b, - 0x36, 0x25, 0xcf, 0x11, 0x17, 0xba, 0x95, 0x21, 0x95, 0x6c, 0x8d, 0x4e, 0x4c, 0x9b, 0x9f, 0xfb, - 0xf7, 0x67, 0xf2, 0x4d, 0x0b, 0x8d, 0xb9, 0xd0, 0xb4, 0x70, 0x72, 0xea, 0x35, 0x2d, 0x9c, 0x32, - 0x4c, 0x3a, 0x73, 0xe4, 0x0f, 0xb0, 0x34, 0x31, 0x9b, 0x11, 0xc7, 0xb0, 0x62, 0xc6, 0x50, 0xd9, - 0x7f, 0x70, 0xa3, 0x4c, 0xa9, 0xff, 0x6b, 0xe8, 0x98, 0x23, 0x11, 0x31, 0x0c, 0x9a, 0x32, 0xf5, - 0xf5, 0xb7, 0x66, 0xb1, 0x4d, 0x85, 0x66, 0x57, 0x36, 0x15, 0x4e, 0x99, 0x4b, 0x4c, 0x85, 0xd3, - 0x9a, 0xb9, 0x33, 0x47, 0x7e, 0x0f, 0x8b, 0xe3, 0xdd, 0x91, 0x7c, 0x3a, 0xee, 0xb6, 0x89, 0xa6, - 0xdb, 0x77, 0x6e, 0x12, 0x29, 0x95, 0xbf, 0x02, 0x18, 0x35, 0x3d, 0x62, 0xc0, 0xcf, 0x44, 0xd3, - 0xed, 0x6f, 0x4c, 0x67, 0x96, 0xaa, 0xfe, 0x04, 0xab, 0x53, 0x3b, 0x0b, 0x31, 0x4a, 0xef, 0xa6, - 0xde, 0xd4, 0xff, 0xf1, 0xad, 0x72, 0xe5, 0x5d, 0x7f, 0x84, 0xa5, 0x09, 0xf8, 0x35, 0xb3, 0x62, - 0x56, 0x57, 0x30, 0xb3, 0x62, 0x26, 0x7e, 0x63, 0xd5, 0x7e, 0x03, 0xdd, 0x0a, 0x30, 0x99, 0x95, - 0x31, 0x0d, 0xfa, 0xcc, 0xca, 0x98, 0x8a, 0x68, 0xce, 0xdc, 0x8e, 0xf5, 0xc4, 0x7a, 0xb6, 0x05, - 0x8b, 0x42, 0xc3, 0xda, 0xa9, 0xd8, 0xd5, 0xdd, 0xe8, 0x19, 0xe0, 0x5b, 0x8f, 0x78, 0x22, 0x93, - 0x93, 0x26, 0xfe, 0xab, 0xfe, 0xd9, 0xff, 0x03, 0x00, 0x00, 0xff, 0xff, 0xf8, 0xd4, 0xa1, 0xe3, - 0xba, 0x16, 0x00, 0x00, + // 2142 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xb4, 0x59, 0x5f, 0x6f, 0xdb, 0xc8, + 0x11, 0x37, 0x25, 0x4b, 0x16, 0x47, 0x52, 0xce, 0x5e, 0x3b, 0x89, 0xa2, 0xc4, 0x89, 0x8f, 0x69, + 0xee, 0x5c, 0x24, 0x70, 0x53, 0xf7, 0x0a, 0xdc, 0xf5, 0xda, 0x07, 0xc7, 0x71, 0xae, 0x69, 0x12, + 0x9f, 0x41, 0x27, 0x87, 0x2b, 0x0a, 0x94, 0xa5, 0xc9, 0xb5, 0xbc, 0x35, 0x45, 0xb2, 0xbb, 0x4b, + 0xff, 0xb9, 0xa7, 0xfb, 0x1c, 0x05, 0xfa, 0xda, 0x4f, 0xd0, 0xc7, 0xa2, 0x2f, 0x45, 0x81, 0x02, + 0x45, 0xbf, 0x44, 0x3f, 0x49, 0xb1, 0xb3, 0x24, 0xb5, 0x14, 0x25, 0xfb, 0x82, 0xc3, 0xbd, 0x71, + 0x67, 0x66, 0x67, 0x67, 0xe7, 0xcf, 0x6f, 0x66, 0x25, 0xe8, 0x1e, 0xb3, 0x88, 0xf2, 0xad, 0x94, + 0x27, 0x32, 0x21, 0x1d, 0x5c, 0x78, 0xe9, 0x91, 0xf3, 0x25, 0xdc, 0x7d, 0x9d, 0x24, 0xa7, 0x59, + 0xfa, 0x9c, 0x71, 0x1a, 0xc8, 0x84, 0x5f, 0xee, 0xc5, 0x92, 0x5f, 0xba, 0xf4, 0x4f, 0x19, 0x15, + 0x92, 0xdc, 0x03, 0x3b, 0x2c, 0x18, 0x03, 0x6b, 0xc3, 0xda, 0xb4, 0xdd, 0x09, 0x81, 0x10, 0x58, + 0x8c, 0xfd, 0x31, 0x1d, 0x34, 0x90, 0x81, 0xdf, 0xce, 0x1e, 0xdc, 0x9b, 0xad, 0x50, 0xa4, 0x49, + 0x2c, 0x28, 0x79, 0x04, 0x2d, 0xaa, 0x08, 0xa8, 0xad, 0xbb, 0xfd, 0xc1, 0x56, 0x61, 0xca, 0x96, + 0x96, 0xd3, 0x5c, 0xe7, 0x1f, 0x16, 0x90, 0xd7, 0x4c, 0x48, 0x45, 0x64, 0x54, 0x7c, 0x37, 0x7b, + 0x6e, 0x41, 0x3b, 0xe5, 0xf4, 0x98, 0x5d, 0xe4, 0x16, 0xe5, 0x2b, 0xf2, 0x04, 0x56, 0x84, 0xf4, + 0xb9, 0x7c, 0xc1, 0x93, 0xf1, 0x0b, 0x16, 0xd1, 0x7d, 0x65, 0x74, 0x13, 0x45, 0xea, 0x0c, 0xb2, + 0x05, 0x84, 0xc5, 0x41, 0x94, 0x09, 0x76, 0x46, 0x0f, 0x0b, 0xee, 0x60, 0x71, 0xc3, 0xda, 0xec, + 0xb8, 0x33, 0x38, 0x64, 0x0d, 0x5a, 0x11, 0x1b, 0x33, 0x39, 0x68, 0x6d, 0x58, 0x9b, 0x7d, 0x57, + 0x2f, 0x9c, 0x5f, 0xc2, 0x6a, 0xc5, 0xfe, 0xf7, 0xbb, 0xfe, 0x5f, 0x1a, 0xd0, 0x42, 0x42, 0xe9, + 0x63, 0x6b, 0xe2, 0x63, 0xf2, 0x21, 0xf4, 0x98, 0xf0, 0x26, 0x8e, 0x68, 0xa0, 0x6d, 0x5d, 0x26, + 0x4a, 0x9f, 0x93, 0xc7, 0xd0, 0x0e, 0x4e, 0xb2, 0xf8, 0x54, 0x0c, 0x9a, 0x1b, 0xcd, 0xcd, 0xee, + 0xf6, 0xea, 0xe4, 0x20, 0x75, 0xd1, 0x5d, 0xc5, 0x73, 0x73, 0x11, 0xf2, 0x29, 0x80, 0x2f, 0x25, + 0x67, 0x47, 0x99, 0xa4, 0x02, 0x6f, 0xda, 0xdd, 0x1e, 0x18, 0x1b, 0x32, 0x41, 0x77, 0x4a, 0xbe, + 0x6b, 0xc8, 0x92, 0xcf, 0xa0, 0x43, 0x2f, 0x24, 0x8d, 0x43, 0x1a, 0x0e, 0x5a, 0x78, 0xd0, 0xfa, + 0xd4, 0x8d, 0xb6, 0xf6, 0x72, 0xbe, 0xbe, 0x5f, 0x29, 0x3e, 0xfc, 0x1c, 0xfa, 0x15, 0x16, 0x59, + 0x86, 0xe6, 0x29, 0x2d, 0xa2, 0xaa, 0x3e, 0x95, 0x67, 0xcf, 0xfc, 0x28, 0xd3, 0x09, 0xd6, 0x73, + 0xf5, 0xe2, 0x17, 0x8d, 0x4f, 0x2d, 0xe7, 0x39, 0xd8, 0x2f, 0xb2, 0x28, 0x2a, 0x37, 0x86, 0x8c, + 0x17, 0x1b, 0x43, 0xc6, 0x27, 0x5e, 0x6e, 0x5c, 0xe9, 0xe5, 0xbf, 0x5b, 0xb0, 0xb2, 0x77, 0x46, + 0x63, 0xb9, 0x9f, 0x48, 0x76, 0xcc, 0x02, 0x5f, 0xb2, 0x24, 0x26, 0x4f, 0xc0, 0x4e, 0xa2, 0xd0, + 0xbb, 0x32, 0x4c, 0x9d, 0x24, 0xca, 0xad, 0x7e, 0x02, 0x76, 0x4c, 0xcf, 0xbd, 0x2b, 0x8f, 0xeb, + 0xc4, 0xf4, 0x5c, 0x4b, 0x3f, 0x84, 0x7e, 0x48, 0x23, 0x2a, 0xa9, 0x57, 0x46, 0x47, 0x85, 0xae, + 0xa7, 0x89, 0xbb, 0x3a, 0x1c, 0x1f, 0xc1, 0x07, 0x4a, 0x65, 0xea, 0x73, 0x1a, 0x4b, 0x2f, 0xf5, + 0xe5, 0x09, 0xc6, 0xc4, 0x76, 0xfb, 0x31, 0x3d, 0x3f, 0x40, 0xea, 0x81, 0x2f, 0x4f, 0x9c, 0xbf, + 0x35, 0xc0, 0x2e, 0x83, 0x49, 0x6e, 0xc3, 0x92, 0x3a, 0xd6, 0x63, 0x61, 0xee, 0x89, 0xb6, 0x5a, + 0xbe, 0x0c, 0x55, 0x55, 0x24, 0xc7, 0xc7, 0x82, 0x4a, 0x34, 0xaf, 0xe9, 0xe6, 0x2b, 0x95, 0x59, + 0x82, 0x7d, 0xa3, 0x0b, 0x61, 0xd1, 0xc5, 0x6f, 0xe5, 0xf1, 0xb1, 0x64, 0x63, 0x8a, 0x07, 0x36, + 0x5d, 0xbd, 0x20, 0xab, 0xd0, 0xa2, 0x9e, 0xf4, 0x47, 0x98, 0xe1, 0xb6, 0xbb, 0x48, 0xdf, 0xfa, + 0x23, 0xf2, 0x23, 0xb8, 0x21, 0x92, 0x8c, 0x07, 0xd4, 0x2b, 0x8e, 0x6d, 0x23, 0xb7, 0xa7, 0xa9, + 0x2f, 0xf4, 0xe1, 0x0e, 0x34, 0x8f, 0x59, 0x38, 0x58, 0x42, 0xc7, 0x2c, 0x57, 0x93, 0xf0, 0x65, + 0xe8, 0x2a, 0x26, 0xf9, 0x09, 0x40, 0xa9, 0x29, 0x1c, 0x74, 0xe6, 0x88, 0xda, 0x85, 0xde, 0x90, + 0xac, 0x03, 0x04, 0x2c, 0x3d, 0xa1, 0xdc, 0x53, 0x09, 0x63, 0x63, 0x72, 0xd8, 0x9a, 0xf2, 0x8a, + 0x5e, 0x2a, 0x36, 0x13, 0xde, 0xe8, 0x1b, 0x96, 0xa6, 0x34, 0x1c, 0x00, 0x7a, 0xd8, 0x66, 0xe2, + 0x0b, 0x4d, 0x70, 0xbe, 0x86, 0x76, 0x6e, 0xdc, 0x5d, 0xb0, 0xcf, 0x92, 0x28, 0x1b, 0x97, 0x4e, + 0xeb, 0xbb, 0x1d, 0x4d, 0x78, 0x19, 0x92, 0x3b, 0x80, 0x28, 0x89, 0x47, 0x34, 0xd0, 0x45, 0xe8, + 0x5f, 0x75, 0xc0, 0x2d, 0x68, 0x07, 0x49, 0x72, 0xca, 0xb4, 0xef, 0x96, 0xdc, 0x7c, 0xe5, 0x7c, + 0xdb, 0x84, 0x1b, 0xd5, 0x62, 0x51, 0x47, 0xa0, 0x16, 0xf4, 0xb4, 0x85, 0x6a, 0x50, 0xed, 0x61, + 0xc5, 0xdb, 0x0d, 0xd3, 0xdb, 0xc5, 0x96, 0x71, 0x12, 0xea, 0x03, 0xfa, 0x7a, 0xcb, 0x9b, 0x24, + 0xa4, 0x2a, 0xd7, 0x33, 0x16, 0x62, 0x78, 0xfa, 0xae, 0xfa, 0x54, 0x94, 0x11, 0x0b, 0x73, 0xf0, + 0x51, 0x9f, 0x68, 0x1e, 0x47, 0xbd, 0x6d, 0x1d, 0x70, 0xbd, 0x52, 0x01, 0x1f, 0x2b, 0xea, 0x92, + 0x8e, 0xa2, 0xfa, 0x26, 0x1b, 0xd0, 0xe5, 0x34, 0x8d, 0xf2, 0xdc, 0x47, 0xe7, 0xdb, 0xae, 0x49, + 0x22, 0xf7, 0x01, 0x82, 0x24, 0x8a, 0x68, 0x80, 0x02, 0x36, 0x0a, 0x18, 0x14, 0x95, 0x77, 0x52, + 0x46, 0x9e, 0xa0, 0x01, 0xba, 0xba, 0xe5, 0xb6, 0xa5, 0x8c, 0x0e, 0x69, 0xa0, 0xee, 0x91, 0x09, + 0xca, 0x3d, 0x84, 0xaf, 0x2e, 0xee, 0xeb, 0x28, 0x02, 0x82, 0xec, 0x3a, 0xc0, 0x88, 0x27, 0x59, + 0xaa, 0xb9, 0xbd, 0x8d, 0xa6, 0x42, 0x72, 0xa4, 0x20, 0xfb, 0x11, 0xdc, 0x10, 0x97, 0xe3, 0x88, + 0xc5, 0xa7, 0x9e, 0xf4, 0xf9, 0x88, 0xca, 0x41, 0x5f, 0x57, 0x40, 0x4e, 0x7d, 0x8b, 0x44, 0x75, + 0xf7, 0x71, 0xf8, 0xf3, 0xc1, 0x0d, 0xcc, 0x00, 0xf5, 0xe9, 0xa4, 0x40, 0x76, 0x39, 0xf5, 0x25, + 0x7d, 0x8f, 0x36, 0xf6, 0xdd, 0xd0, 0x82, 0xdc, 0x84, 0x76, 0xe2, 0xd1, 0x8b, 0x20, 0xca, 0x8b, + 0xb6, 0x95, 0xec, 0x5d, 0x04, 0x91, 0xf3, 0x18, 0x56, 0x2b, 0x27, 0xe6, 0x40, 0xbf, 0x06, 0x2d, + 0xca, 0x79, 0x52, 0xc0, 0x92, 0x5e, 0x38, 0xbf, 0x05, 0xf2, 0x2e, 0x0d, 0x7f, 0x08, 0xf3, 0x9c, + 0x9b, 0xb0, 0x5a, 0x51, 0xad, 0xed, 0x70, 0xbe, 0xb5, 0x60, 0x6d, 0x27, 0x4d, 0x69, 0x1c, 0xbe, + 0x4d, 0xde, 0xe3, 0xd0, 0x75, 0x00, 0x54, 0xeb, 0x19, 0x0d, 0xde, 0x46, 0x0a, 0xc6, 0xe7, 0x7d, + 0xda, 0x8b, 0x73, 0x1b, 0x6e, 0x4e, 0x59, 0x90, 0xdb, 0xf6, 0x2f, 0x0b, 0xc8, 0x73, 0x44, 0xbe, + 0xef, 0x37, 0x74, 0x28, 0x2c, 0x52, 0x0d, 0x51, 0x23, 0x6b, 0xe8, 0x4b, 0x3f, 0x6f, 0xd7, 0x3d, + 0x26, 0xb4, 0xfe, 0xe7, 0xbe, 0xf4, 0xf3, 0xb6, 0xc9, 0x69, 0x90, 0x71, 0xd5, 0xc1, 0xb1, 0x64, + 0xb0, 0x6d, 0xba, 0x05, 0x89, 0x7c, 0x02, 0xb7, 0xd8, 0x28, 0x4e, 0x38, 0x9d, 0x88, 0x79, 0x3a, + 0x8c, 0x6d, 0x14, 0x5e, 0xd3, 0xdc, 0x72, 0xc3, 0x1e, 0x46, 0xf5, 0x31, 0xac, 0x56, 0xae, 0x71, + 0x65, 0x0a, 0xfc, 0xd9, 0x82, 0xc1, 0x8e, 0x4c, 0xc6, 0x2c, 0x70, 0xa9, 0x32, 0xbe, 0x72, 0xf5, + 0x87, 0xd0, 0x57, 0xbd, 0x67, 0xfa, 0xfa, 0xbd, 0x24, 0x0a, 0x27, 0xbd, 0xfd, 0x0e, 0xa8, 0xf6, + 0x63, 0x46, 0x66, 0x29, 0x89, 0x42, 0x8c, 0xcb, 0x43, 0x50, 0x3d, 0xc2, 0xd8, 0xaf, 0xa7, 0x9c, + 0x5e, 0x4c, 0xcf, 0x2b, 0xfb, 0x95, 0x10, 0xee, 0xd7, 0x8d, 0x65, 0x29, 0xa6, 0xe7, 0x6a, 0xbf, + 0x73, 0x17, 0xee, 0xcc, 0xb0, 0x2d, 0x0f, 0xd7, 0xbf, 0x2d, 0x58, 0xdd, 0x11, 0x82, 0x8d, 0xe2, + 0xaf, 0x10, 0x24, 0x0b, 0xa3, 0xd7, 0xa0, 0x15, 0x24, 0x59, 0x2c, 0xd1, 0xd8, 0x96, 0xab, 0x17, + 0x53, 0xb8, 0xd1, 0xa8, 0xe1, 0xc6, 0x14, 0xf2, 0x34, 0xeb, 0xc8, 0x63, 0x20, 0xcb, 0x62, 0x05, + 0x59, 0x1e, 0x40, 0x57, 0x05, 0xd9, 0x0b, 0x68, 0x2c, 0x29, 0xcf, 0xbb, 0x12, 0x28, 0xd2, 0x2e, + 0x52, 0x94, 0x80, 0xd9, 0x3d, 0x75, 0x63, 0x82, 0x74, 0xd2, 0x3a, 0xff, 0xa7, 0xaa, 0xa2, 0x72, + 0x95, 0x3c, 0x66, 0x73, 0xbb, 0xa8, 0x02, 0x5e, 0x1e, 0xe5, 0xf7, 0x50, 0x9f, 0xaa, 0x44, 0xd2, + 0xec, 0x28, 0x62, 0x81, 0xa7, 0x18, 0xda, 0x7e, 0x5b, 0x53, 0xde, 0xf1, 0x68, 0xe2, 0x95, 0x45, + 0xd3, 0x2b, 0x04, 0x16, 0xfd, 0x4c, 0x9e, 0x14, 0x9d, 0x54, 0x7d, 0x4f, 0x79, 0xaa, 0x7d, 0x9d, + 0xa7, 0x96, 0xea, 0x9e, 0x2a, 0x33, 0xad, 0x63, 0x66, 0xda, 0x27, 0xb0, 0xaa, 0x47, 0xf1, 0x6a, + 0xb8, 0xd6, 0x01, 0xca, 0xae, 0x27, 0x06, 0x96, 0x86, 0xde, 0xa2, 0xed, 0x09, 0xe7, 0x57, 0x60, + 0xbf, 0x4e, 0xb4, 0x5e, 0x41, 0x9e, 0x82, 0x1d, 0x15, 0x0b, 0x14, 0xed, 0x6e, 0x93, 0x49, 0xa9, + 0x17, 0x72, 0xee, 0x44, 0xc8, 0xf9, 0x1c, 0x3a, 0x05, 0xb9, 0xf0, 0x99, 0x35, 0xcf, 0x67, 0x8d, + 0x29, 0x9f, 0x39, 0xff, 0xb4, 0x60, 0xad, 0x6a, 0x72, 0x1e, 0x96, 0x77, 0xd0, 0x2f, 0x8f, 0xf0, + 0xc6, 0x7e, 0x9a, 0xdb, 0xf2, 0xd4, 0xb4, 0xa5, 0xbe, 0xad, 0x34, 0x50, 0xbc, 0xf1, 0x53, 0x9d, + 0xcb, 0xbd, 0xc8, 0x20, 0x0d, 0xdf, 0xc2, 0x4a, 0x4d, 0x64, 0xc6, 0x1c, 0xfa, 0x63, 0x73, 0x0e, + 0xad, 0x80, 0x5d, 0xb9, 0xdb, 0x1c, 0x4e, 0x3f, 0x83, 0xdb, 0x1a, 0x0e, 0x76, 0xcb, 0x18, 0x16, + 0xbe, 0xaf, 0x86, 0xda, 0x9a, 0x0e, 0xb5, 0x33, 0x84, 0x41, 0x7d, 0x6b, 0x5e, 0x7e, 0x23, 0x58, + 0x39, 0x94, 0xbe, 0x64, 0x42, 0xb2, 0xa0, 0x7c, 0x10, 0x4d, 0xe5, 0x86, 0x75, 0x5d, 0xff, 0xae, + 0xd7, 0xe1, 0x32, 0x34, 0xa5, 0x2c, 0xf2, 0x57, 0x7d, 0xaa, 0x28, 0x10, 0xf3, 0xa4, 0x3c, 0x06, + 0x3f, 0xc0, 0x51, 0x2a, 0x1f, 0x64, 0x22, 0xfd, 0x48, 0xcf, 0x47, 0x8b, 0x38, 0x1f, 0xd9, 0x48, + 0xc1, 0x01, 0x49, 0x8f, 0x10, 0xa1, 0xe6, 0xb6, 0xf4, 0xf4, 0xa4, 0x08, 0xc8, 0x5c, 0x07, 0xc0, + 0x52, 0xd5, 0x55, 0xd6, 0xd6, 0x7b, 0x15, 0x65, 0x57, 0x11, 0x9c, 0xfb, 0x70, 0xef, 0x0b, 0x2a, + 0x55, 0x37, 0xe2, 0xbb, 0x49, 0x7c, 0xcc, 0x46, 0x19, 0xf7, 0x8d, 0x50, 0x38, 0xff, 0xb1, 0x60, + 0x7d, 0x8e, 0x40, 0x7e, 0xe1, 0x01, 0x2c, 0x8d, 0x7d, 0x21, 0x29, 0x2f, 0xaa, 0xa4, 0x58, 0x4e, + 0xbb, 0xa2, 0x71, 0x9d, 0x2b, 0x9a, 0x35, 0x57, 0xdc, 0x84, 0xf6, 0xd8, 0xbf, 0xf0, 0xc6, 0x47, + 0xf9, 0x28, 0xd7, 0x1a, 0xfb, 0x17, 0x6f, 0x8e, 0x10, 0xd9, 0x18, 0xf7, 0x8e, 0xb2, 0xe0, 0x94, + 0x4a, 0x51, 0x22, 0x1b, 0xe3, 0xcf, 0x34, 0x05, 0x67, 0x3b, 0x1c, 0x74, 0x11, 0x06, 0x3a, 0x6e, + 0xbe, 0x72, 0xce, 0x61, 0x70, 0x98, 0x1d, 0x89, 0x80, 0xb3, 0x23, 0xfa, 0x86, 0x4a, 0x5f, 0x81, + 0x61, 0x91, 0x23, 0x0f, 0xa0, 0x1b, 0x44, 0x4c, 0xa1, 0xa1, 0xf1, 0x92, 0x04, 0x4d, 0xc2, 0xae, + 0x81, 0x70, 0x29, 0x4f, 0xbc, 0xca, 0xe3, 0x19, 0x14, 0xe9, 0x40, 0x3f, 0xa0, 0xef, 0x40, 0x47, + 0xb0, 0x38, 0xa0, 0x5e, 0xac, 0x5f, 0x2c, 0x4d, 0x77, 0x09, 0xd7, 0xfb, 0x42, 0xb5, 0xb3, 0x3b, + 0x33, 0x4e, 0xce, 0x5d, 0x78, 0x75, 0x2b, 0xff, 0x0d, 0x10, 0x7a, 0x86, 0x76, 0x19, 0xef, 0xaf, + 0xbc, 0xc8, 0xee, 0x1a, 0x63, 0xce, 0xf4, 0x13, 0xcd, 0x5d, 0xa1, 0xb5, 0x57, 0xdb, 0x2a, 0xb4, + 0xa4, 0x98, 0xd8, 0xb7, 0x28, 0xc5, 0xbe, 0x70, 0x7c, 0x05, 0x46, 0x23, 0x5d, 0xd6, 0xa5, 0x80, + 0x35, 0x11, 0x20, 0x4f, 0x80, 0xa4, 0x3e, 0x97, 0x4c, 0xa9, 0x50, 0x93, 0xbe, 0x77, 0xe2, 0x8b, + 0x13, 0xb4, 0xa0, 0xe5, 0x2e, 0x97, 0x9c, 0x57, 0xf4, 0xf2, 0xd7, 0xbe, 0x38, 0x51, 0xe0, 0x8d, + 0xc3, 0x45, 0x13, 0xe7, 0x4d, 0xfc, 0x76, 0x28, 0xac, 0xbd, 0xa2, 0x34, 0xdd, 0x4d, 0xe2, 0x98, + 0x06, 0x92, 0x86, 0x85, 0xd3, 0x67, 0xbd, 0xdb, 0xef, 0x82, 0x3d, 0xe2, 0x69, 0xe0, 0xa5, 0x09, + 0xd7, 0x8f, 0xb1, 0xbe, 0xdb, 0x51, 0x84, 0x83, 0x84, 0xe3, 0xd4, 0xc3, 0xa9, 0x7e, 0xe3, 0xe8, + 0xa9, 0xca, 0x76, 0x27, 0x04, 0x35, 0x43, 0x4d, 0x1d, 0x93, 0xa3, 0xc2, 0x4f, 0x15, 0xc8, 0x07, + 0xbe, 0xa4, 0xcf, 0x78, 0x72, 0x4a, 0x79, 0x71, 0xfc, 0x10, 0x3a, 0xc5, 0xe6, 0xdc, 0x84, 0x72, + 0xed, 0xfc, 0x17, 0x51, 0xd6, 0xdc, 0x33, 0x19, 0x58, 0x8e, 0x93, 0x2c, 0xd6, 0xad, 0xaf, 0xe3, + 0xea, 0x05, 0xd9, 0x33, 0x0d, 0x6b, 0x20, 0xee, 0x7e, 0x3c, 0x85, 0x80, 0x53, 0x8a, 0xb6, 0xdc, + 0x5c, 0xde, 0xb8, 0xc1, 0xf0, 0x6b, 0xe8, 0x14, 0x64, 0x35, 0xde, 0xa3, 0x23, 0xfc, 0x30, 0xe4, + 0x54, 0x08, 0x2a, 0x72, 0x1b, 0xfb, 0x8a, 0xba, 0x53, 0x10, 0x95, 0x58, 0xb1, 0x3f, 0xaf, 0x72, + 0x1d, 0x99, 0x7e, 0x41, 0xc5, 0x4a, 0xdf, 0xfe, 0x2b, 0x40, 0xef, 0x90, 0xfa, 0xe7, 0x94, 0x86, + 0x58, 0xcd, 0x64, 0x54, 0x74, 0x91, 0xea, 0x6f, 0x50, 0xe4, 0xd1, 0x74, 0xbb, 0x98, 0xf9, 0xa3, + 0xd7, 0xf0, 0xa3, 0xeb, 0xc4, 0x72, 0xd7, 0x2f, 0x90, 0x7d, 0xe8, 0x1a, 0x3f, 0xf2, 0x90, 0x7b, + 0xc6, 0xc6, 0xda, 0x6f, 0x57, 0xc3, 0xf5, 0x39, 0xdc, 0x42, 0xdb, 0x53, 0x8b, 0xbc, 0x86, 0xae, + 0xf1, 0x96, 0x30, 0xf5, 0xd5, 0x1f, 0x35, 0xa6, 0xbe, 0x19, 0x0f, 0x10, 0x67, 0x41, 0x69, 0x33, + 0x5e, 0x04, 0xa6, 0xb6, 0xfa, 0x1b, 0xc4, 0xd4, 0x36, 0xeb, 0x19, 0xb1, 0x40, 0x5c, 0xe8, 0x57, + 0xa6, 0x78, 0x72, 0x7f, 0xb2, 0x63, 0xd6, 0x03, 0x63, 0xf8, 0x60, 0x2e, 0xdf, 0xb4, 0xd0, 0x18, + 0x9c, 0x4d, 0x0b, 0xeb, 0xcf, 0x02, 0xd3, 0xc2, 0x19, 0xd3, 0xb6, 0xb3, 0x40, 0x7e, 0x0f, 0x2b, + 0xb5, 0xe1, 0x95, 0x38, 0x86, 0x15, 0x73, 0xa6, 0xee, 0xe1, 0xc3, 0x2b, 0x65, 0x4a, 0xfd, 0x5f, + 0x42, 0xcf, 0x9c, 0x19, 0x89, 0x61, 0xd0, 0x8c, 0xb1, 0x78, 0x78, 0x7f, 0x1e, 0xdb, 0x54, 0x68, + 0x8e, 0x2d, 0xa6, 0xc2, 0x19, 0x83, 0x9b, 0xa9, 0x70, 0xd6, 0xb4, 0xe3, 0x2c, 0x90, 0xdf, 0xc1, + 0xf2, 0xf4, 0xf8, 0x40, 0x3e, 0x9c, 0x76, 0x5b, 0x6d, 0x2a, 0x19, 0x3a, 0x57, 0x89, 0x94, 0xca, + 0x5f, 0x02, 0x4c, 0xa6, 0x02, 0x62, 0xe0, 0x73, 0x6d, 0x2a, 0x19, 0xde, 0x9b, 0xcd, 0x2c, 0x55, + 0xfd, 0x11, 0x6e, 0xce, 0x6c, 0xbd, 0xc4, 0x28, 0xbd, 0xab, 0x9a, 0xf7, 0xf0, 0xe3, 0x6b, 0xe5, + 0xca, 0xb3, 0xfe, 0x00, 0x2b, 0xb5, 0xfe, 0x64, 0x66, 0xc5, 0xbc, 0xb6, 0x69, 0x66, 0xc5, 0xdc, + 0x06, 0x87, 0x55, 0xfb, 0x15, 0xf4, 0x2b, 0xd8, 0x6c, 0x56, 0xc6, 0xac, 0xde, 0x60, 0x56, 0xc6, + 0x6c, 0x50, 0x5f, 0xd8, 0xb4, 0x9e, 0x5a, 0x3a, 0x3d, 0x26, 0xe8, 0x5a, 0x4d, 0x8f, 0x1a, 0xe4, + 0x57, 0xd3, 0xa3, 0x0e, 0xca, 0xce, 0xc2, 0xb3, 0xfb, 0xb0, 0x2c, 0x34, 0x4e, 0x1e, 0x8b, 0x2d, + 0xdd, 0xff, 0x9f, 0x01, 0x3a, 0xef, 0x80, 0x27, 0x32, 0x39, 0x6a, 0xe3, 0xbf, 0x03, 0x3f, 0xfb, + 0x7f, 0x00, 0x00, 0x00, 0xff, 0xff, 0x32, 0x17, 0xbf, 0x34, 0x2c, 0x18, 0x00, 0x00, } diff --git a/weed/pb/messaging.proto b/weed/pb/messaging.proto index 56f60b212..689c22d29 100644 --- a/weed/pb/messaging.proto +++ b/weed/pb/messaging.proto @@ -15,6 +15,9 @@ service SeaweedMessaging { rpc Publish (stream PublishRequest) returns (stream PublishResponse) { } + rpc DeleteTopic (DeleteTopicRequest) returns (DeleteTopicResponse) { + } + rpc ConfigureTopic (ConfigureTopicRequest) returns (ConfigureTopicResponse) { } @@ -47,6 +50,7 @@ message SubscriberMessage { int64 message_id = 1; } AckMessage ack = 2; + bool is_close = 3; } message Message { @@ -54,11 +58,11 @@ message Message { bytes key = 2; // Message key bytes value = 3; // Message payload map headers = 4; // Message headers + bool is_close = 5; } message BrokerMessage { Message data = 1; - bool is_close = 2; } message PublishRequest { @@ -80,6 +84,14 @@ message PublishResponse { string new_broker = 1; } RedirectMessage redirect = 2; + bool is_closed = 3; +} + +message DeleteTopicRequest { + string namespace = 1; + string topic = 2; +} +message DeleteTopicResponse { } message ConfigureTopicRequest { diff --git a/weed/pb/messaging_pb/messaging.pb.go b/weed/pb/messaging_pb/messaging.pb.go index 809fd3ded..f42e2c2db 100644 --- a/weed/pb/messaging_pb/messaging.pb.go +++ b/weed/pb/messaging_pb/messaging.pb.go @@ -14,6 +14,8 @@ It has these top-level messages: BrokerMessage PublishRequest PublishResponse + DeleteTopicRequest + DeleteTopicResponse ConfigureTopicRequest ConfigureTopicResponse GetTopicConfigurationRequest @@ -93,12 +95,13 @@ func (x TopicConfiguration_Partitioning) String() string { return proto.EnumName(TopicConfiguration_Partitioning_name, int32(x)) } func (TopicConfiguration_Partitioning) EnumDescriptor() ([]byte, []int) { - return fileDescriptor0, []int{11, 0} + return fileDescriptor0, []int{13, 0} } type SubscriberMessage struct { - Init *SubscriberMessage_InitMessage `protobuf:"bytes,1,opt,name=init" json:"init,omitempty"` - Ack *SubscriberMessage_AckMessage `protobuf:"bytes,2,opt,name=ack" json:"ack,omitempty"` + Init *SubscriberMessage_InitMessage `protobuf:"bytes,1,opt,name=init" json:"init,omitempty"` + Ack *SubscriberMessage_AckMessage `protobuf:"bytes,2,opt,name=ack" json:"ack,omitempty"` + IsClose bool `protobuf:"varint,3,opt,name=is_close,json=isClose" json:"is_close,omitempty"` } func (m *SubscriberMessage) Reset() { *m = SubscriberMessage{} } @@ -120,6 +123,13 @@ func (m *SubscriberMessage) GetAck() *SubscriberMessage_AckMessage { return nil } +func (m *SubscriberMessage) GetIsClose() bool { + if m != nil { + return m.IsClose + } + return false +} + type SubscriberMessage_InitMessage struct { Namespace string `protobuf:"bytes,1,opt,name=namespace" json:"namespace,omitempty"` Topic string `protobuf:"bytes,2,opt,name=topic" json:"topic,omitempty"` @@ -199,6 +209,7 @@ type Message struct { Key []byte `protobuf:"bytes,2,opt,name=key,proto3" json:"key,omitempty"` Value []byte `protobuf:"bytes,3,opt,name=value,proto3" json:"value,omitempty"` Headers map[string][]byte `protobuf:"bytes,4,rep,name=headers" json:"headers,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value,proto3"` + IsClose bool `protobuf:"varint,5,opt,name=is_close,json=isClose" json:"is_close,omitempty"` } func (m *Message) Reset() { *m = Message{} } @@ -234,9 +245,15 @@ func (m *Message) GetHeaders() map[string][]byte { return nil } +func (m *Message) GetIsClose() bool { + if m != nil { + return m.IsClose + } + return false +} + type BrokerMessage struct { - Data *Message `protobuf:"bytes,1,opt,name=data" json:"data,omitempty"` - IsClose bool `protobuf:"varint,2,opt,name=is_close,json=isClose" json:"is_close,omitempty"` + Data *Message `protobuf:"bytes,1,opt,name=data" json:"data,omitempty"` } func (m *BrokerMessage) Reset() { *m = BrokerMessage{} } @@ -251,13 +268,6 @@ func (m *BrokerMessage) GetData() *Message { return nil } -func (m *BrokerMessage) GetIsClose() bool { - if m != nil { - return m.IsClose - } - return false -} - type PublishRequest struct { Init *PublishRequest_InitMessage `protobuf:"bytes,1,opt,name=init" json:"init,omitempty"` Data *Message `protobuf:"bytes,2,opt,name=data" json:"data,omitempty"` @@ -317,6 +327,7 @@ func (m *PublishRequest_InitMessage) GetPartition() int32 { type PublishResponse struct { Config *PublishResponse_ConfigMessage `protobuf:"bytes,1,opt,name=config" json:"config,omitempty"` Redirect *PublishResponse_RedirectMessage `protobuf:"bytes,2,opt,name=redirect" json:"redirect,omitempty"` + IsClosed bool `protobuf:"varint,3,opt,name=is_closed,json=isClosed" json:"is_closed,omitempty"` } func (m *PublishResponse) Reset() { *m = PublishResponse{} } @@ -338,6 +349,13 @@ func (m *PublishResponse) GetRedirect() *PublishResponse_RedirectMessage { return nil } +func (m *PublishResponse) GetIsClosed() bool { + if m != nil { + return m.IsClosed + } + return false +} + type PublishResponse_ConfigMessage struct { PartitionCount int32 `protobuf:"varint,1,opt,name=partition_count,json=partitionCount" json:"partition_count,omitempty"` } @@ -374,6 +392,38 @@ func (m *PublishResponse_RedirectMessage) GetNewBroker() string { return "" } +type DeleteTopicRequest struct { + Namespace string `protobuf:"bytes,1,opt,name=namespace" json:"namespace,omitempty"` + Topic string `protobuf:"bytes,2,opt,name=topic" json:"topic,omitempty"` +} + +func (m *DeleteTopicRequest) Reset() { *m = DeleteTopicRequest{} } +func (m *DeleteTopicRequest) String() string { return proto.CompactTextString(m) } +func (*DeleteTopicRequest) ProtoMessage() {} +func (*DeleteTopicRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{5} } + +func (m *DeleteTopicRequest) GetNamespace() string { + if m != nil { + return m.Namespace + } + return "" +} + +func (m *DeleteTopicRequest) GetTopic() string { + if m != nil { + return m.Topic + } + return "" +} + +type DeleteTopicResponse struct { +} + +func (m *DeleteTopicResponse) Reset() { *m = DeleteTopicResponse{} } +func (m *DeleteTopicResponse) String() string { return proto.CompactTextString(m) } +func (*DeleteTopicResponse) ProtoMessage() {} +func (*DeleteTopicResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{6} } + type ConfigureTopicRequest struct { Namespace string `protobuf:"bytes,1,opt,name=namespace" json:"namespace,omitempty"` Topic string `protobuf:"bytes,2,opt,name=topic" json:"topic,omitempty"` @@ -383,7 +433,7 @@ type ConfigureTopicRequest struct { func (m *ConfigureTopicRequest) Reset() { *m = ConfigureTopicRequest{} } func (m *ConfigureTopicRequest) String() string { return proto.CompactTextString(m) } func (*ConfigureTopicRequest) ProtoMessage() {} -func (*ConfigureTopicRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{5} } +func (*ConfigureTopicRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{7} } func (m *ConfigureTopicRequest) GetNamespace() string { if m != nil { @@ -412,7 +462,7 @@ type ConfigureTopicResponse struct { func (m *ConfigureTopicResponse) Reset() { *m = ConfigureTopicResponse{} } func (m *ConfigureTopicResponse) String() string { return proto.CompactTextString(m) } func (*ConfigureTopicResponse) ProtoMessage() {} -func (*ConfigureTopicResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{6} } +func (*ConfigureTopicResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{8} } type GetTopicConfigurationRequest struct { Namespace string `protobuf:"bytes,1,opt,name=namespace" json:"namespace,omitempty"` @@ -422,7 +472,7 @@ type GetTopicConfigurationRequest struct { func (m *GetTopicConfigurationRequest) Reset() { *m = GetTopicConfigurationRequest{} } func (m *GetTopicConfigurationRequest) String() string { return proto.CompactTextString(m) } func (*GetTopicConfigurationRequest) ProtoMessage() {} -func (*GetTopicConfigurationRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{7} } +func (*GetTopicConfigurationRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{9} } func (m *GetTopicConfigurationRequest) GetNamespace() string { if m != nil { @@ -445,7 +495,7 @@ type GetTopicConfigurationResponse struct { func (m *GetTopicConfigurationResponse) Reset() { *m = GetTopicConfigurationResponse{} } func (m *GetTopicConfigurationResponse) String() string { return proto.CompactTextString(m) } func (*GetTopicConfigurationResponse) ProtoMessage() {} -func (*GetTopicConfigurationResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{8} } +func (*GetTopicConfigurationResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{10} } func (m *GetTopicConfigurationResponse) GetConfiguration() *TopicConfiguration { if m != nil { @@ -463,7 +513,7 @@ type FindBrokerRequest struct { func (m *FindBrokerRequest) Reset() { *m = FindBrokerRequest{} } func (m *FindBrokerRequest) String() string { return proto.CompactTextString(m) } func (*FindBrokerRequest) ProtoMessage() {} -func (*FindBrokerRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{9} } +func (*FindBrokerRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{11} } func (m *FindBrokerRequest) GetNamespace() string { if m != nil { @@ -493,7 +543,7 @@ type FindBrokerResponse struct { func (m *FindBrokerResponse) Reset() { *m = FindBrokerResponse{} } func (m *FindBrokerResponse) String() string { return proto.CompactTextString(m) } func (*FindBrokerResponse) ProtoMessage() {} -func (*FindBrokerResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{10} } +func (*FindBrokerResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{12} } func (m *FindBrokerResponse) GetBroker() string { if m != nil { @@ -513,7 +563,7 @@ type TopicConfiguration struct { func (m *TopicConfiguration) Reset() { *m = TopicConfiguration{} } func (m *TopicConfiguration) String() string { return proto.CompactTextString(m) } func (*TopicConfiguration) ProtoMessage() {} -func (*TopicConfiguration) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{11} } +func (*TopicConfiguration) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{13} } func (m *TopicConfiguration) GetPartitionCount() int32 { if m != nil { @@ -561,6 +611,8 @@ func init() { proto.RegisterType((*PublishResponse)(nil), "messaging_pb.PublishResponse") proto.RegisterType((*PublishResponse_ConfigMessage)(nil), "messaging_pb.PublishResponse.ConfigMessage") proto.RegisterType((*PublishResponse_RedirectMessage)(nil), "messaging_pb.PublishResponse.RedirectMessage") + proto.RegisterType((*DeleteTopicRequest)(nil), "messaging_pb.DeleteTopicRequest") + proto.RegisterType((*DeleteTopicResponse)(nil), "messaging_pb.DeleteTopicResponse") proto.RegisterType((*ConfigureTopicRequest)(nil), "messaging_pb.ConfigureTopicRequest") proto.RegisterType((*ConfigureTopicResponse)(nil), "messaging_pb.ConfigureTopicResponse") proto.RegisterType((*GetTopicConfigurationRequest)(nil), "messaging_pb.GetTopicConfigurationRequest") @@ -585,6 +637,7 @@ const _ = grpc.SupportPackageIsVersion4 type SeaweedMessagingClient interface { Subscribe(ctx context.Context, opts ...grpc.CallOption) (SeaweedMessaging_SubscribeClient, error) Publish(ctx context.Context, opts ...grpc.CallOption) (SeaweedMessaging_PublishClient, error) + DeleteTopic(ctx context.Context, in *DeleteTopicRequest, opts ...grpc.CallOption) (*DeleteTopicResponse, error) ConfigureTopic(ctx context.Context, in *ConfigureTopicRequest, opts ...grpc.CallOption) (*ConfigureTopicResponse, error) GetTopicConfiguration(ctx context.Context, in *GetTopicConfigurationRequest, opts ...grpc.CallOption) (*GetTopicConfigurationResponse, error) FindBroker(ctx context.Context, in *FindBrokerRequest, opts ...grpc.CallOption) (*FindBrokerResponse, error) @@ -660,6 +713,15 @@ func (x *seaweedMessagingPublishClient) Recv() (*PublishResponse, error) { return m, nil } +func (c *seaweedMessagingClient) DeleteTopic(ctx context.Context, in *DeleteTopicRequest, opts ...grpc.CallOption) (*DeleteTopicResponse, error) { + out := new(DeleteTopicResponse) + err := grpc.Invoke(ctx, "/messaging_pb.SeaweedMessaging/DeleteTopic", in, out, c.cc, opts...) + if err != nil { + return nil, err + } + return out, nil +} + func (c *seaweedMessagingClient) ConfigureTopic(ctx context.Context, in *ConfigureTopicRequest, opts ...grpc.CallOption) (*ConfigureTopicResponse, error) { out := new(ConfigureTopicResponse) err := grpc.Invoke(ctx, "/messaging_pb.SeaweedMessaging/ConfigureTopic", in, out, c.cc, opts...) @@ -692,6 +754,7 @@ func (c *seaweedMessagingClient) FindBroker(ctx context.Context, in *FindBrokerR type SeaweedMessagingServer interface { Subscribe(SeaweedMessaging_SubscribeServer) error Publish(SeaweedMessaging_PublishServer) error + DeleteTopic(context.Context, *DeleteTopicRequest) (*DeleteTopicResponse, error) ConfigureTopic(context.Context, *ConfigureTopicRequest) (*ConfigureTopicResponse, error) GetTopicConfiguration(context.Context, *GetTopicConfigurationRequest) (*GetTopicConfigurationResponse, error) FindBroker(context.Context, *FindBrokerRequest) (*FindBrokerResponse, error) @@ -753,6 +816,24 @@ func (x *seaweedMessagingPublishServer) Recv() (*PublishRequest, error) { return m, nil } +func _SeaweedMessaging_DeleteTopic_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(DeleteTopicRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(SeaweedMessagingServer).DeleteTopic(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/messaging_pb.SeaweedMessaging/DeleteTopic", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(SeaweedMessagingServer).DeleteTopic(ctx, req.(*DeleteTopicRequest)) + } + return interceptor(ctx, in, info, handler) +} + func _SeaweedMessaging_ConfigureTopic_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(ConfigureTopicRequest) if err := dec(in); err != nil { @@ -811,6 +892,10 @@ var _SeaweedMessaging_serviceDesc = grpc.ServiceDesc{ ServiceName: "messaging_pb.SeaweedMessaging", HandlerType: (*SeaweedMessagingServer)(nil), Methods: []grpc.MethodDesc{ + { + MethodName: "DeleteTopic", + Handler: _SeaweedMessaging_DeleteTopic_Handler, + }, { MethodName: "ConfigureTopic", Handler: _SeaweedMessaging_ConfigureTopic_Handler, @@ -844,65 +929,68 @@ var _SeaweedMessaging_serviceDesc = grpc.ServiceDesc{ func init() { proto.RegisterFile("messaging.proto", fileDescriptor0) } var fileDescriptor0 = []byte{ - // 952 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xb4, 0x56, 0xdd, 0x6e, 0xe3, 0x44, - 0x14, 0xae, 0x9d, 0x34, 0x3f, 0x27, 0x3f, 0xcd, 0x1e, 0xd1, 0x55, 0x30, 0x2d, 0x04, 0x2f, 0x82, - 0x40, 0x21, 0xaa, 0xc2, 0x4d, 0x59, 0xad, 0xb4, 0x6a, 0xa3, 0x2e, 0x1b, 0xd1, 0x76, 0xc3, 0x24, - 0x5c, 0x22, 0xcb, 0xb1, 0x67, 0xd3, 0x51, 0x9d, 0xb1, 0xf1, 0x38, 0x5b, 0xf5, 0x9a, 0x6b, 0xae, - 0x78, 0x15, 0x5e, 0x80, 0x67, 0xe0, 0x02, 0x89, 0xa7, 0x41, 0x1e, 0xff, 0xc4, 0x4e, 0xb2, 0xe9, - 0x52, 0xb4, 0x77, 0x9e, 0x33, 0xdf, 0xf9, 0xce, 0x77, 0xce, 0x9c, 0x33, 0x63, 0xd8, 0x9b, 0x53, - 0x21, 0xcc, 0x19, 0xe3, 0xb3, 0x9e, 0xe7, 0xbb, 0x81, 0x8b, 0xf5, 0xd4, 0x60, 0x78, 0x53, 0xfd, - 0xd7, 0x22, 0x3c, 0x1a, 0x2f, 0xa6, 0xc2, 0xf2, 0xd9, 0x94, 0xfa, 0x97, 0x72, 0x8b, 0xe2, 0x73, - 0x28, 0x32, 0xce, 0x82, 0xb6, 0xd2, 0x51, 0xba, 0xb5, 0xfe, 0x51, 0x2f, 0xeb, 0xd2, 0x5b, 0x83, - 0xf7, 0x86, 0x9c, 0x05, 0xf1, 0x37, 0x91, 0x8e, 0xf8, 0x0c, 0x0a, 0xa6, 0x75, 0xd3, 0x56, 0xa5, - 0xff, 0x57, 0xf7, 0xf9, 0x9f, 0x5a, 0x37, 0x89, 0x7b, 0xe8, 0xa6, 0xfd, 0xa9, 0x42, 0x2d, 0xc3, - 0x89, 0x07, 0x50, 0xe5, 0xe6, 0x9c, 0x0a, 0xcf, 0xb4, 0xa8, 0xd4, 0x54, 0x25, 0x4b, 0x03, 0x7e, - 0x00, 0xbb, 0x81, 0xeb, 0x31, 0x4b, 0x46, 0xab, 0x92, 0x68, 0x11, 0xfa, 0x78, 0xa6, 0x1f, 0xb0, - 0x80, 0xb9, 0xbc, 0x5d, 0xe8, 0x28, 0xdd, 0x5d, 0xb2, 0x34, 0xa0, 0x01, 0x0d, 0x11, 0x98, 0x7e, - 0x30, 0x72, 0x45, 0x84, 0x28, 0x76, 0x94, 0x6e, 0xb3, 0xff, 0xdd, 0x7f, 0xc8, 0xb4, 0x37, 0xce, - 0x12, 0x90, 0x3c, 0x1f, 0x76, 0xa0, 0x16, 0xb0, 0x39, 0x15, 0x81, 0x39, 0xf7, 0xae, 0x44, 0x7b, - 0xb7, 0xa3, 0x74, 0x0b, 0x24, 0x6b, 0xc2, 0x27, 0xd0, 0x10, 0x29, 0xbf, 0xc1, 0xec, 0x76, 0x49, - 0xca, 0xaf, 0x2f, 0x8d, 0x43, 0x5b, 0x3f, 0x81, 0x46, 0x2e, 0x0c, 0x02, 0x94, 0x2e, 0x4e, 0x27, - 0xe7, 0xe3, 0x49, 0x6b, 0x07, 0xeb, 0x50, 0x39, 0x3f, 0x25, 0x17, 0xc3, 0x70, 0xa5, 0x60, 0x03, - 0xaa, 0x93, 0xe1, 0xe5, 0xf9, 0x78, 0x72, 0x7a, 0x39, 0x6a, 0xa9, 0xda, 0x11, 0xc0, 0xb2, 0xac, - 0x78, 0x08, 0x10, 0x65, 0x46, 0xc3, 0x48, 0x8a, 0x54, 0x53, 0x8d, 0x2d, 0x43, 0x5b, 0xff, 0x4b, - 0x81, 0x72, 0x02, 0xfd, 0x1c, 0x1a, 0xf4, 0x0d, 0xe5, 0x81, 0x11, 0x8a, 0x35, 0xb8, 0x88, 0xd0, - 0x67, 0xea, 0xb1, 0x42, 0x6a, 0x72, 0x63, 0xc2, 0xe6, 0xf4, 0x4a, 0x60, 0x0b, 0x0a, 0x37, 0xf4, - 0x4e, 0x16, 0xbd, 0x4e, 0xc2, 0xcf, 0xf0, 0x20, 0xde, 0x98, 0xce, 0x82, 0xca, 0x72, 0xd7, 0x49, - 0xb4, 0xc0, 0x67, 0x50, 0xbe, 0xa6, 0xa6, 0x4d, 0x7d, 0xd1, 0x2e, 0x76, 0x0a, 0xdd, 0x5a, 0x5f, - 0xcf, 0x17, 0x39, 0x29, 0xe7, 0xcb, 0x08, 0x74, 0xce, 0x03, 0xff, 0x8e, 0x24, 0x2e, 0xda, 0x53, - 0xa8, 0x67, 0x37, 0x92, 0xa8, 0x51, 0x13, 0xe4, 0xa3, 0xaa, 0x99, 0xa8, 0x4f, 0xd5, 0x13, 0x45, - 0xff, 0x09, 0x1a, 0x67, 0xbe, 0x7b, 0xb3, 0x6c, 0xeb, 0x2f, 0xa1, 0x68, 0x9b, 0x81, 0x19, 0xb7, - 0xf5, 0xfe, 0x46, 0x1d, 0x44, 0x42, 0xf0, 0x43, 0xa8, 0x30, 0x61, 0x58, 0x8e, 0x2b, 0x22, 0xe2, - 0x0a, 0x29, 0x33, 0x31, 0x08, 0x97, 0xfa, 0x3f, 0x0a, 0x34, 0x47, 0x8b, 0xa9, 0xc3, 0xc4, 0x35, - 0xa1, 0xbf, 0x2c, 0xa8, 0x08, 0xdb, 0x3d, 0x3b, 0x2f, 0xdd, 0x3c, 0x71, 0x1e, 0xbb, 0x61, 0x58, - 0x12, 0x59, 0xea, 0xbd, 0xb2, 0x34, 0xe3, 0x3d, 0x0f, 0x86, 0xfe, 0x9b, 0x0a, 0x7b, 0xa9, 0x60, - 0xe1, 0xb9, 0x5c, 0x50, 0x1c, 0x40, 0xc9, 0x72, 0xf9, 0x6b, 0x36, 0xdb, 0x7c, 0x1f, 0xac, 0xc0, - 0x7b, 0x03, 0x89, 0x4d, 0x74, 0xc7, 0xae, 0x38, 0x84, 0x8a, 0x4f, 0x6d, 0xe6, 0x53, 0x2b, 0x88, - 0x13, 0xfd, 0x66, 0x3b, 0x0d, 0x89, 0xd1, 0x09, 0x51, 0xea, 0xae, 0x9d, 0x40, 0x23, 0x17, 0x03, - 0xbf, 0x80, 0xbd, 0x34, 0x03, 0xc3, 0x72, 0x17, 0x3c, 0x3a, 0x89, 0x5d, 0xd2, 0x4c, 0xcd, 0x83, - 0xd0, 0xaa, 0x1d, 0xc3, 0xde, 0x0a, 0x6d, 0x38, 0x19, 0x9c, 0xde, 0x1a, 0x53, 0xd9, 0x28, 0x69, - 0x0d, 0xe9, 0x6d, 0xd4, 0x39, 0xfa, 0xef, 0x0a, 0xec, 0x47, 0xc1, 0x16, 0x3e, 0x9d, 0x84, 0x05, - 0x4c, 0xce, 0xfc, 0x21, 0xb5, 0x7f, 0x01, 0x0d, 0x2b, 0x26, 0x33, 0xd3, 0xfa, 0xd7, 0xfa, 0x9d, - 0x7c, 0x25, 0x64, 0x98, 0x41, 0x16, 0x47, 0xf2, 0x6e, 0x7a, 0x1b, 0x1e, 0xaf, 0x8a, 0x8a, 0xaa, - 0xa6, 0x13, 0x38, 0xf8, 0x9e, 0x06, 0x1b, 0x18, 0x1e, 0xae, 0x5a, 0x9f, 0xc1, 0xe1, 0x5b, 0x38, - 0xe3, 0x06, 0x59, 0x4b, 0x4b, 0x79, 0x58, 0x5a, 0x16, 0x3c, 0x7a, 0xc1, 0xb8, 0x1d, 0x95, 0xfe, - 0xff, 0xd4, 0x59, 0x83, 0x8a, 0x67, 0xfa, 0xd9, 0x16, 0x4f, 0xd7, 0xfa, 0xd7, 0x80, 0xd9, 0x20, - 0x71, 0x0a, 0x8f, 0xa1, 0x94, 0x6b, 0x81, 0x78, 0xa5, 0xff, 0xa1, 0x02, 0xae, 0x0b, 0x7f, 0xe7, - 0x8e, 0xc3, 0x8f, 0x01, 0x2c, 0xd7, 0x71, 0xa8, 0x25, 0xb5, 0x44, 0x22, 0x33, 0x96, 0xf0, 0x9d, - 0xf0, 0xa9, 0xe7, 0x30, 0x6b, 0xd9, 0x0f, 0x55, 0x92, 0x35, 0xe1, 0xa7, 0x50, 0x67, 0xc2, 0x08, - 0x7c, 0x93, 0x0b, 0x46, 0x79, 0x20, 0x5f, 0xaa, 0x0a, 0xa9, 0x31, 0x31, 0x49, 0x4c, 0xf8, 0x0a, - 0x6a, 0x51, 0x58, 0x97, 0x33, 0x3e, 0x93, 0x8f, 0x4d, 0x73, 0x75, 0xbc, 0xd6, 0x93, 0xe8, 0x8d, - 0x12, 0xa9, 0x8c, 0xcf, 0x48, 0x96, 0x41, 0x7f, 0x0e, 0xf5, 0xec, 0x26, 0x22, 0x34, 0xaf, 0x5c, - 0x7e, 0xb5, 0x70, 0x9c, 0x1f, 0xe8, 0xdd, 0x4b, 0x53, 0x5c, 0xb7, 0x76, 0xb0, 0x06, 0xe5, 0x64, - 0xa1, 0x60, 0x13, 0x80, 0xb8, 0x0b, 0x6e, 0x13, 0x77, 0xca, 0x78, 0x4b, 0xed, 0xff, 0x5d, 0x80, - 0xd6, 0x98, 0x9a, 0xb7, 0x94, 0xda, 0x97, 0x89, 0x0a, 0x7c, 0x05, 0xd5, 0xf4, 0x45, 0xc5, 0x4f, - 0xee, 0x79, 0x6a, 0xb5, 0x8f, 0xf2, 0x80, 0xdc, 0x4d, 0xae, 0xef, 0x74, 0x95, 0x63, 0x05, 0x2f, - 0xa0, 0x1c, 0xdf, 0x1a, 0x78, 0xb0, 0xed, 0xce, 0xd5, 0x0e, 0xb7, 0x5e, 0x35, 0x31, 0xdb, 0xcf, - 0xd0, 0xcc, 0x0f, 0x15, 0x3e, 0xc9, 0xbb, 0x6d, 0xbc, 0x07, 0xb4, 0xcf, 0xb6, 0x83, 0x92, 0x10, - 0xe8, 0xc3, 0xfe, 0xc6, 0x29, 0xc2, 0x95, 0xdf, 0xa3, 0x6d, 0xe3, 0xab, 0x1d, 0xbd, 0x13, 0x36, - 0x8d, 0xf9, 0x23, 0xc0, 0xb2, 0xd7, 0x57, 0x4b, 0xbe, 0x36, 0x6a, 0x5a, 0xe7, 0xed, 0x80, 0x84, - 0xf2, 0x4c, 0x87, 0x96, 0x88, 0x0e, 0xf6, 0xb5, 0xe8, 0x59, 0x4e, 0xd8, 0x7f, 0x67, 0xcd, 0xf4, - 0x8c, 0x47, 0xe1, 0x2f, 0xe6, 0xb4, 0x24, 0xff, 0x34, 0xbf, 0xfd, 0x37, 0x00, 0x00, 0xff, 0xff, - 0xad, 0x6b, 0x26, 0x8c, 0x7c, 0x0a, 0x00, 0x00, + // 1002 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xb4, 0x56, 0xdd, 0x6e, 0xe3, 0x54, + 0x10, 0xae, 0xdd, 0xfc, 0x8e, 0x93, 0x34, 0x3b, 0xd0, 0x55, 0xf0, 0xb6, 0x90, 0xf5, 0x22, 0x08, + 0x14, 0xa2, 0x2a, 0xdc, 0x94, 0x6a, 0xa5, 0x55, 0x1b, 0xba, 0x34, 0xa2, 0xed, 0x86, 0x93, 0xdc, + 0x22, 0xcb, 0xb1, 0xcf, 0xa6, 0x47, 0x75, 0x8e, 0x8d, 0x8f, 0xb3, 0x55, 0x9f, 0x83, 0x7b, 0x1e, + 0x00, 0x89, 0x3b, 0x5e, 0x80, 0xd7, 0xe0, 0x21, 0x78, 0x06, 0xe4, 0xdf, 0xd8, 0x49, 0x36, 0x5d, + 0xb6, 0xda, 0xbb, 0x9c, 0xc9, 0x37, 0x33, 0xdf, 0x99, 0xf9, 0x66, 0x8e, 0x61, 0x67, 0x46, 0x85, + 0x30, 0xa6, 0x8c, 0x4f, 0xbb, 0xae, 0xe7, 0xf8, 0x0e, 0xd6, 0x52, 0x83, 0xee, 0x4e, 0xb4, 0xdf, + 0x0b, 0xf0, 0x68, 0x34, 0x9f, 0x08, 0xd3, 0x63, 0x13, 0xea, 0x5d, 0x86, 0x7f, 0x51, 0x7c, 0x01, + 0x05, 0xc6, 0x99, 0xdf, 0x92, 0xda, 0x52, 0x47, 0xe9, 0x1d, 0x74, 0xb3, 0x2e, 0xdd, 0x15, 0x78, + 0x77, 0xc0, 0x99, 0x1f, 0xff, 0x26, 0xa1, 0x23, 0x3e, 0x87, 0x6d, 0xc3, 0xbc, 0x69, 0xc9, 0xa1, + 0xff, 0xd7, 0xf7, 0xf9, 0x9f, 0x98, 0x37, 0x89, 0x7b, 0xe0, 0x86, 0x9f, 0x40, 0x85, 0x09, 0xdd, + 0xb4, 0x1d, 0x41, 0x5b, 0xdb, 0x6d, 0xa9, 0x53, 0x21, 0x65, 0x26, 0xfa, 0xc1, 0x51, 0xfd, 0x5b, + 0x06, 0x25, 0x93, 0x0e, 0xf7, 0xa0, 0xca, 0x8d, 0x19, 0x15, 0xae, 0x61, 0xd2, 0x90, 0x6e, 0x95, + 0x2c, 0x0c, 0xf8, 0x31, 0x14, 0x7d, 0xc7, 0x65, 0x66, 0x48, 0xa4, 0x4a, 0xa2, 0x43, 0xe0, 0xe3, + 0x1a, 0x9e, 0xcf, 0x7c, 0xe6, 0xf0, 0x30, 0x7e, 0x91, 0x2c, 0x0c, 0xa8, 0x43, 0x5d, 0xf8, 0x86, + 0xe7, 0x0f, 0x1d, 0x11, 0x21, 0x0a, 0x6d, 0xa9, 0xd3, 0xe8, 0x7d, 0xff, 0x3f, 0x8a, 0xd0, 0x1d, + 0x65, 0x03, 0x90, 0x7c, 0x3c, 0x6c, 0x83, 0xe2, 0xb3, 0x19, 0x15, 0xbe, 0x31, 0x73, 0xaf, 0x44, + 0xab, 0xd8, 0x96, 0x3a, 0xdb, 0x24, 0x6b, 0xc2, 0x67, 0x50, 0x17, 0x69, 0x7c, 0x9d, 0x59, 0xad, + 0x52, 0x48, 0xbf, 0xb6, 0x30, 0x0e, 0x2c, 0xed, 0x08, 0xea, 0xb9, 0x34, 0x08, 0x50, 0xba, 0x38, + 0x19, 0x9f, 0x8d, 0xc6, 0xcd, 0x2d, 0xac, 0x41, 0xe5, 0xec, 0x84, 0x5c, 0x0c, 0x82, 0x93, 0x84, + 0x75, 0xa8, 0x8e, 0x07, 0x97, 0x67, 0xa3, 0xf1, 0xc9, 0xe5, 0xb0, 0x29, 0xab, 0x07, 0x00, 0x8b, + 0x8a, 0xe3, 0x3e, 0x40, 0x74, 0x33, 0x1a, 0x64, 0x92, 0x42, 0x36, 0xd5, 0xd8, 0x32, 0xb0, 0xb4, + 0x7f, 0x25, 0x28, 0x27, 0xd0, 0x2f, 0xa0, 0x4e, 0xdf, 0x50, 0xee, 0xeb, 0x01, 0x59, 0x9d, 0x8b, + 0x08, 0x7d, 0x2a, 0x1f, 0x4a, 0x44, 0x09, 0xff, 0x18, 0xb3, 0x19, 0xbd, 0x12, 0xd8, 0x84, 0xed, + 0x1b, 0x7a, 0x17, 0x16, 0xbd, 0x46, 0x82, 0x9f, 0x41, 0x23, 0xde, 0x18, 0xf6, 0x3c, 0x6a, 0x67, + 0x8d, 0x44, 0x07, 0x7c, 0x0e, 0xe5, 0x6b, 0x6a, 0x58, 0xd4, 0x13, 0xad, 0x42, 0x7b, 0xbb, 0xa3, + 0xf4, 0xb4, 0x7c, 0x91, 0x93, 0x72, 0x9e, 0x47, 0xa0, 0x33, 0xee, 0x7b, 0x77, 0x24, 0x71, 0xc9, + 0xa9, 0xa4, 0x98, 0x57, 0xc9, 0x31, 0xd4, 0xb2, 0x3e, 0x09, 0xa1, 0x48, 0x1f, 0x79, 0x42, 0x72, + 0x86, 0xd0, 0xb1, 0x7c, 0x24, 0x69, 0xc7, 0x50, 0x3f, 0xf5, 0x9c, 0x9b, 0xc5, 0x30, 0x7c, 0x05, + 0x05, 0xcb, 0xf0, 0x8d, 0x78, 0x18, 0x76, 0xd7, 0x52, 0x24, 0x21, 0x44, 0xfb, 0x47, 0x82, 0xc6, + 0x70, 0x3e, 0xb1, 0x99, 0xb8, 0x26, 0xf4, 0xd7, 0x39, 0x15, 0xc1, 0x24, 0x64, 0x47, 0xa9, 0x93, + 0xf7, 0xce, 0x63, 0xd7, 0xcc, 0x51, 0x92, 0x5b, 0xbe, 0x37, 0xb7, 0xaa, 0x7f, 0xe0, 0xc1, 0xd0, + 0xfe, 0x90, 0x61, 0x27, 0x25, 0x2c, 0x5c, 0x87, 0x0b, 0x8a, 0x7d, 0x28, 0x99, 0x0e, 0x7f, 0xcd, + 0xa6, 0xeb, 0x57, 0xc5, 0x12, 0xbc, 0xdb, 0x0f, 0xb1, 0x09, 0xef, 0xd8, 0x15, 0x07, 0x50, 0xf1, + 0xa8, 0xc5, 0x3c, 0x6a, 0xfa, 0xf1, 0x45, 0xbf, 0xdd, 0x1c, 0x86, 0xc4, 0xe8, 0x24, 0x50, 0xea, + 0x8e, 0x4f, 0xa0, 0x9a, 0x68, 0xc2, 0x8a, 0x57, 0x47, 0x25, 0x16, 0x85, 0xa5, 0x1e, 0x41, 0x3d, + 0x47, 0x00, 0xbf, 0x84, 0x9d, 0xf4, 0x7a, 0xba, 0xe9, 0xcc, 0x79, 0xd4, 0xa6, 0x22, 0x69, 0xa4, + 0xe6, 0x7e, 0x60, 0x55, 0x0f, 0x61, 0x67, 0x29, 0x67, 0x30, 0x36, 0x9c, 0xde, 0xea, 0x93, 0x50, + 0x2a, 0x69, 0x81, 0xe9, 0x6d, 0xa4, 0x1d, 0xed, 0x1c, 0xf0, 0x07, 0x6a, 0x53, 0x9f, 0x8e, 0x83, + 0xca, 0x26, 0x62, 0x78, 0x8f, 0xa6, 0x68, 0xbb, 0xf0, 0x51, 0x2e, 0x52, 0x54, 0x03, 0xed, 0x37, + 0x09, 0x76, 0xa3, 0xdb, 0xcc, 0xbd, 0x07, 0x27, 0xc1, 0x97, 0x50, 0x37, 0xe3, 0x60, 0x46, 0xda, + 0x7d, 0xa5, 0xd7, 0xce, 0xf7, 0x21, 0x4c, 0xd3, 0xcf, 0xe2, 0x48, 0xde, 0x4d, 0x6b, 0xc1, 0xe3, + 0x65, 0x52, 0x31, 0x5f, 0x02, 0x7b, 0x3f, 0x52, 0x7f, 0x4d, 0x84, 0x07, 0x94, 0x66, 0x0a, 0xfb, + 0x6f, 0x89, 0x19, 0xcb, 0x73, 0xe5, 0x5a, 0xd2, 0xfb, 0x5d, 0xcb, 0x84, 0x47, 0x2f, 0x19, 0xb7, + 0xa2, 0xde, 0x3e, 0xa4, 0xce, 0x2a, 0x54, 0x5c, 0xc3, 0xcb, 0x0e, 0x58, 0x7a, 0xd6, 0xbe, 0x01, + 0xcc, 0x26, 0x89, 0xaf, 0xf0, 0x18, 0x4a, 0x39, 0x8d, 0xc5, 0x27, 0xed, 0x2f, 0x19, 0x70, 0x95, + 0xf8, 0x3b, 0x4b, 0x1a, 0x3f, 0x05, 0x30, 0x1d, 0xdb, 0xa6, 0x66, 0xc8, 0x25, 0x22, 0x99, 0xb1, + 0x04, 0xaf, 0x94, 0x47, 0x5d, 0x9b, 0x99, 0x0b, 0x3d, 0x54, 0x49, 0xd6, 0x84, 0x4f, 0xa1, 0xc6, + 0x84, 0xee, 0x7b, 0x06, 0x17, 0x8c, 0x72, 0x3f, 0x7c, 0x27, 0x2b, 0x44, 0x61, 0x62, 0x9c, 0x98, + 0xf0, 0x15, 0x28, 0x51, 0x5a, 0x87, 0x33, 0x3e, 0x0d, 0xb7, 0x74, 0x63, 0x79, 0xb8, 0x57, 0x2f, + 0xd1, 0x1d, 0x26, 0x54, 0x19, 0x9f, 0x92, 0x6c, 0x04, 0xed, 0x05, 0xd4, 0xb2, 0x7f, 0x22, 0x42, + 0xe3, 0xca, 0xe1, 0x57, 0x73, 0xdb, 0xfe, 0x89, 0xde, 0x9d, 0x1b, 0xe2, 0xba, 0xb9, 0x85, 0x0a, + 0x94, 0x93, 0x83, 0x84, 0x0d, 0x00, 0xe2, 0xcc, 0xb9, 0x45, 0x9c, 0x09, 0xe3, 0x4d, 0xb9, 0xf7, + 0x67, 0x01, 0x9a, 0x23, 0x6a, 0xdc, 0x52, 0x6a, 0x5d, 0x26, 0x2c, 0xf0, 0x15, 0x54, 0xd3, 0xf7, + 0x1c, 0x3f, 0xbb, 0xe7, 0xa1, 0x57, 0x9f, 0xe4, 0x01, 0xb9, 0xc7, 0x42, 0xdb, 0xea, 0x48, 0x87, + 0x12, 0x5e, 0x40, 0x39, 0xde, 0x59, 0xb8, 0xb7, 0x69, 0xe3, 0xab, 0xfb, 0x1b, 0x17, 0x5d, 0x1c, + 0x6d, 0x0c, 0x4a, 0x66, 0x03, 0xe0, 0x92, 0x7a, 0x57, 0xd7, 0x8c, 0xfa, 0x74, 0x03, 0x22, 0x89, + 0x8c, 0xbf, 0x40, 0x23, 0x3f, 0xaa, 0xf8, 0x2c, 0xef, 0xb6, 0x76, 0xbb, 0xa8, 0x9f, 0x6f, 0x06, + 0xa5, 0xe1, 0x3d, 0xd8, 0x5d, 0x3b, 0x9b, 0xb8, 0xf4, 0x35, 0xb8, 0x69, 0x29, 0xa8, 0x07, 0xef, + 0x84, 0x4d, 0x73, 0xfe, 0x0c, 0xb0, 0x98, 0xa0, 0xe5, 0x46, 0xae, 0x0c, 0xb0, 0xda, 0x7e, 0x3b, + 0x20, 0x09, 0x79, 0xaa, 0x41, 0x53, 0x44, 0x72, 0x79, 0x2d, 0xba, 0xa6, 0x1d, 0xa8, 0xfa, 0xb4, + 0x91, 0x2a, 0x67, 0x18, 0x7c, 0x51, 0x4f, 0x4a, 0xe1, 0x87, 0xf5, 0x77, 0xff, 0x05, 0x00, 0x00, + 0xff, 0xff, 0x7f, 0x62, 0xba, 0x48, 0x6b, 0x0b, 0x00, 0x00, } diff --git a/weed/server/filer_grpc_server.go b/weed/server/filer_grpc_server.go index 266030f5d..901f798f0 100644 --- a/weed/server/filer_grpc_server.go +++ b/weed/server/filer_grpc_server.go @@ -390,8 +390,12 @@ func (fs *FilerServer) KeepConnected(stream filer_pb.SeaweedFiler_KeepConnectedS } clientName := fmt.Sprintf("%s:%d", req.Name, req.GrpcPort) + m := make(map[string]bool) + for _, tp := range req.Resources { + m[tp] = true + } fs.brokersLock.Lock() - fs.brokers[clientName] = true + fs.brokers[clientName] = m glog.V(0).Infof("+ broker %v", clientName) fs.brokersLock.Unlock() @@ -417,3 +421,35 @@ func (fs *FilerServer) KeepConnected(stream filer_pb.SeaweedFiler_KeepConnectedS } } + +func (fs *FilerServer) LocateBroker(ctx context.Context, req *filer_pb.LocateBrokerRequest) (resp *filer_pb.LocateBrokerResponse, err error) { + + resp = &filer_pb.LocateBrokerResponse{} + + fs.brokersLock.Lock() + defer fs.brokersLock.Unlock() + + var localBrokers []*filer_pb.LocateBrokerResponse_Resource + + for b, m := range fs.brokers { + if _, found := m[req.Resource]; found { + resp.Found = true + resp.Resources = []*filer_pb.LocateBrokerResponse_Resource{ + { + GrpcAddresses: b, + ResourceCount: int32(len(m)), + }, + } + return + } + localBrokers = append(localBrokers, &filer_pb.LocateBrokerResponse_Resource{ + GrpcAddresses: b, + ResourceCount: int32(len(m)), + }) + } + + resp.Resources = localBrokers + + return resp, nil + +} diff --git a/weed/server/filer_grpc_server_listen.go b/weed/server/filer_grpc_server_listen.go index e45068f39..848a1fc3a 100644 --- a/weed/server/filer_grpc_server_listen.go +++ b/weed/server/filer_grpc_server_listen.go @@ -82,7 +82,7 @@ func (fs *FilerServer) SubscribeMetadata(req *filer_pb.SubscribeMetadataRequest, lastReadTime = time.Unix(0, processedTsNs) } - _, err := fs.filer.MetaLogBuffer.LoopProcessLogData(lastReadTime, func() bool { + err := fs.filer.MetaLogBuffer.LoopProcessLogData(lastReadTime, func() bool { fs.listenersLock.Lock() fs.listenersCond.Wait() fs.listenersLock.Unlock() diff --git a/weed/server/filer_server.go b/weed/server/filer_server.go index 0a54ea97d..10b607dfe 100644 --- a/weed/server/filer_server.go +++ b/weed/server/filer_server.go @@ -64,7 +64,7 @@ type FilerServer struct { listenersLock sync.Mutex listenersCond *sync.Cond - brokers map[string]bool + brokers map[string]map[string]bool brokersLock sync.Mutex } @@ -73,7 +73,7 @@ func NewFilerServer(defaultMux, readonlyMux *http.ServeMux, option *FilerOption) fs = &FilerServer{ option: option, grpcDialOption: security.LoadClientTLS(util.GetViper(), "grpc.filer"), - brokers: make(map[string]bool), + brokers: make(map[string]map[string]bool), } fs.listenersCond = sync.NewCond(&fs.listenersLock) diff --git a/weed/util/log_buffer/log_read.go b/weed/util/log_buffer/log_read.go index 6339d9d77..a2345acfb 100644 --- a/weed/util/log_buffer/log_read.go +++ b/weed/util/log_buffer/log_read.go @@ -12,8 +12,9 @@ import ( ) func (logBuffer *LogBuffer) LoopProcessLogData( - startTreadTime time.Time, waitForDataFn func() bool, - eachLogDataFn func(logEntry *filer_pb.LogEntry) error) (processed int64, err error) { + startTreadTime time.Time, + waitForDataFn func() bool, + eachLogDataFn func(logEntry *filer_pb.LogEntry) error) (err error) { // loop through all messages var bytesBuf *bytes.Buffer lastReadTime := startTreadTime @@ -66,7 +67,6 @@ func (logBuffer *LogBuffer) LoopProcessLogData( pos += 4 + int(size) batchSize++ - processed++ } // fmt.Printf("sent message ts[%d,%d] size %d\n", startReadTime.UnixNano(), lastReadTime.UnixNano(), batchSize) From 07d7abe428186c7771f51589cc397ecefa6453d2 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sat, 9 May 2020 00:31:34 -0700 Subject: [PATCH 03/24] add deleteTopic, refactoring --- weed/messaging/broker/broker_grpc_server.go | 20 +++++- .../broker/broker_grpc_server_subscribe.go | 2 +- .../{pub_sub_chan.go => pub_chan.go} | 54 ---------------- weed/messaging/msgclient/sub_chan.go | 63 +++++++++++++++++++ 4 files changed, 83 insertions(+), 56 deletions(-) rename weed/messaging/msgclient/{pub_sub_chan.go => pub_chan.go} (58%) create mode 100644 weed/messaging/msgclient/sub_chan.go diff --git a/weed/messaging/broker/broker_grpc_server.go b/weed/messaging/broker/broker_grpc_server.go index 32dab6813..305213622 100644 --- a/weed/messaging/broker/broker_grpc_server.go +++ b/weed/messaging/broker/broker_grpc_server.go @@ -2,7 +2,10 @@ package broker import ( "context" + "fmt" + "github.com/chrislusf/seaweedfs/weed/filer2" + "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "github.com/chrislusf/seaweedfs/weed/pb/messaging_pb" ) @@ -11,9 +14,24 @@ func (broker *MessageBroker) ConfigureTopic(c context.Context, request *messagin } func (broker *MessageBroker) DeleteTopic(c context.Context, request *messaging_pb.DeleteTopicRequest) (*messaging_pb.DeleteTopicResponse, error) { - panic("implement me") + resp := &messaging_pb.DeleteTopicResponse{} + dir, entry := genTopicDirEntry(request.Namespace, request.Topic) + if exists, err := filer_pb.Exists(broker, dir, entry, true); err != nil { + return nil, err + } else if exists { + err = filer_pb.Remove(broker, dir, entry, true, true, true) + } + return resp, nil } func (broker *MessageBroker) GetTopicConfiguration(c context.Context, request *messaging_pb.GetTopicConfigurationRequest) (*messaging_pb.GetTopicConfigurationResponse, error) { panic("implement me") } + +func genTopicDir(namespace, topic string) string { + return fmt.Sprintf("%s/%s/%s", filer2.TopicsDir, namespace, topic) +} + +func genTopicDirEntry(namespace, topic string) (dir, entry string) { + return fmt.Sprintf("%s/%s/%s", filer2.TopicsDir, namespace), topic +} diff --git a/weed/messaging/broker/broker_grpc_server_subscribe.go b/weed/messaging/broker/broker_grpc_server_subscribe.go index 761129e80..f8fd16a14 100644 --- a/weed/messaging/broker/broker_grpc_server_subscribe.go +++ b/weed/messaging/broker/broker_grpc_server_subscribe.go @@ -119,7 +119,7 @@ func (broker *MessageBroker) readPersistedLogBuffer(tp *TopicPartition, startTim sizeBuf := make([]byte, 4) startTsNs := startTime.UnixNano() - topicDir := fmt.Sprintf("/topics/%s/%s", tp.Namespace, tp.Topic) + topicDir := genTopicDir(tp.Namespace, tp.Topic) partitionSuffix := fmt.Sprintf(".part%02d", tp.Partition) return filer_pb.List(broker, topicDir, "", func(dayEntry *filer_pb.Entry, isLast bool) error { diff --git a/weed/messaging/msgclient/pub_sub_chan.go b/weed/messaging/msgclient/pub_chan.go similarity index 58% rename from weed/messaging/msgclient/pub_sub_chan.go rename to weed/messaging/msgclient/pub_chan.go index a11240080..ccf301a6a 100644 --- a/weed/messaging/msgclient/pub_sub_chan.go +++ b/weed/messaging/msgclient/pub_chan.go @@ -3,7 +3,6 @@ package msgclient import ( "io" "log" - "time" "google.golang.org/grpc" @@ -63,56 +62,3 @@ func (pc *PubChannel) Close() error { } return nil } - -type SubChannel struct { - ch chan []byte - stream messaging_pb.SeaweedMessaging_SubscribeClient -} - -func (mc *MessagingClient) NewSubChannel(chanName string) (*SubChannel, error) { - tp := broker.TopicPartition{ - Namespace: "chan", - Topic: chanName, - Partition: 0, - } - grpcConnection, err := mc.findBroker(tp) - if err != nil { - return nil, err - } - sc, err := setupSubscriberClient(grpcConnection, "", "chan", chanName, 0, time.Unix(0, 0)) - if err != nil { - return nil, err - } - - t := &SubChannel{ - ch: make(chan []byte), - stream: sc, - } - - go func() { - for { - resp, subErr := t.stream.Recv() - if subErr == io.EOF { - return - } - if subErr != nil { - log.Printf("fail to receive from netchan %s: %v", chanName, subErr) - return - } - if resp.Data.IsClose { - t.stream.Send(&messaging_pb.SubscriberMessage{ - IsClose: true, - }) - close(t.ch) - return - } - t.ch <- resp.Data.Value - } - }() - - return t, nil -} - -func (sc *SubChannel) Channel() chan []byte { - return sc.ch -} diff --git a/weed/messaging/msgclient/sub_chan.go b/weed/messaging/msgclient/sub_chan.go new file mode 100644 index 000000000..edd4d1049 --- /dev/null +++ b/weed/messaging/msgclient/sub_chan.go @@ -0,0 +1,63 @@ +package msgclient + +import ( + "io" + "log" + "time" + + "github.com/chrislusf/seaweedfs/weed/messaging/broker" + "github.com/chrislusf/seaweedfs/weed/pb/messaging_pb" +) + +type SubChannel struct { + ch chan []byte + stream messaging_pb.SeaweedMessaging_SubscribeClient +} + +func (mc *MessagingClient) NewSubChannel(chanName string) (*SubChannel, error) { + tp := broker.TopicPartition{ + Namespace: "chan", + Topic: chanName, + Partition: 0, + } + grpcConnection, err := mc.findBroker(tp) + if err != nil { + return nil, err + } + sc, err := setupSubscriberClient(grpcConnection, "", "chan", chanName, 0, time.Unix(0, 0)) + if err != nil { + return nil, err + } + + t := &SubChannel{ + ch: make(chan []byte), + stream: sc, + } + + go func() { + for { + resp, subErr := t.stream.Recv() + if subErr == io.EOF { + return + } + if subErr != nil { + log.Printf("fail to receive from netchan %s: %v", chanName, subErr) + return + } + if resp.Data.IsClose { + t.stream.Send(&messaging_pb.SubscriberMessage{ + IsClose: true, + }) + close(t.ch) + return + } + t.ch <- resp.Data.Value + } + }() + + return t, nil +} + +func (sc *SubChannel) Channel() chan []byte { + return sc.ch +} From d693e7741852db9f77b542d9e07a3a6620448a83 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sat, 9 May 2020 00:43:53 -0700 Subject: [PATCH 04/24] add pub sub md5 --- weed/messaging/broker/broker_grpc_server.go | 2 +- weed/messaging/msgclient/pub_chan.go | 14 +++++++++++++- weed/messaging/msgclient/sub_chan.go | 17 +++++++++++++---- 3 files changed, 27 insertions(+), 6 deletions(-) diff --git a/weed/messaging/broker/broker_grpc_server.go b/weed/messaging/broker/broker_grpc_server.go index 305213622..6918a28a6 100644 --- a/weed/messaging/broker/broker_grpc_server.go +++ b/weed/messaging/broker/broker_grpc_server.go @@ -33,5 +33,5 @@ func genTopicDir(namespace, topic string) string { } func genTopicDirEntry(namespace, topic string) (dir, entry string) { - return fmt.Sprintf("%s/%s/%s", filer2.TopicsDir, namespace), topic + return fmt.Sprintf("%s/%s", filer2.TopicsDir, namespace), topic } diff --git a/weed/messaging/msgclient/pub_chan.go b/weed/messaging/msgclient/pub_chan.go index ccf301a6a..9bc88f7c0 100644 --- a/weed/messaging/msgclient/pub_chan.go +++ b/weed/messaging/msgclient/pub_chan.go @@ -1,6 +1,8 @@ package msgclient import ( + "crypto/md5" + "hash" "io" "log" @@ -13,6 +15,7 @@ import ( type PubChannel struct { client messaging_pb.SeaweedMessaging_PublishClient grpcConnection *grpc.ClientConn + md5hash hash.Hash } func (mc *MessagingClient) NewPubChannel(chanName string) (*PubChannel, error) { @@ -32,15 +35,20 @@ func (mc *MessagingClient) NewPubChannel(chanName string) (*PubChannel, error) { return &PubChannel{ client: pc, grpcConnection: grpcConnection, + md5hash: md5.New(), }, nil } func (pc *PubChannel) Publish(m []byte) error { - return pc.client.Send(&messaging_pb.PublishRequest{ + err := pc.client.Send(&messaging_pb.PublishRequest{ Data: &messaging_pb.Message{ Value: m, }, }) + if err == nil { + pc.md5hash.Write(m) + } + return err } func (pc *PubChannel) Close() error { @@ -62,3 +70,7 @@ func (pc *PubChannel) Close() error { } return nil } + +func (pc *PubChannel) Md5() []byte { + return pc.md5hash.Sum(nil) +} diff --git a/weed/messaging/msgclient/sub_chan.go b/weed/messaging/msgclient/sub_chan.go index edd4d1049..aae5c0c71 100644 --- a/weed/messaging/msgclient/sub_chan.go +++ b/weed/messaging/msgclient/sub_chan.go @@ -1,6 +1,8 @@ package msgclient import ( + "crypto/md5" + "hash" "io" "log" "time" @@ -10,8 +12,9 @@ import ( ) type SubChannel struct { - ch chan []byte - stream messaging_pb.SeaweedMessaging_SubscribeClient + ch chan []byte + stream messaging_pb.SeaweedMessaging_SubscribeClient + md5hash hash.Hash } func (mc *MessagingClient) NewSubChannel(chanName string) (*SubChannel, error) { @@ -30,8 +33,9 @@ func (mc *MessagingClient) NewSubChannel(chanName string) (*SubChannel, error) { } t := &SubChannel{ - ch: make(chan []byte), - stream: sc, + ch: make(chan []byte), + stream: sc, + md5hash: md5.New(), } go func() { @@ -51,6 +55,7 @@ func (mc *MessagingClient) NewSubChannel(chanName string) (*SubChannel, error) { close(t.ch) return } + t.md5hash.Write(resp.Data.Value) t.ch <- resp.Data.Value } }() @@ -61,3 +66,7 @@ func (mc *MessagingClient) NewSubChannel(chanName string) (*SubChannel, error) { func (sc *SubChannel) Channel() chan []byte { return sc.ch } + +func (sc *SubChannel) Md5() []byte { + return sc.md5hash.Sum(nil) +} From 6bf3eb69cb9abd02e1d63ecdee485d198a3cab9a Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sun, 10 May 2020 03:48:35 -0700 Subject: [PATCH 05/24] async chan write read, no write for closed chan --- .../broker/broker_grpc_server_publish.go | 22 +++++++++++++++++++ .../broker/broker_grpc_server_subscribe.go | 1 + weed/messaging/broker/topic_lock.go | 3 ++- weed/messaging/msgclient/sub_chan.go | 2 +- weed/util/log_buffer/log_buffer.go | 11 +++++++--- 5 files changed, 34 insertions(+), 5 deletions(-) diff --git a/weed/messaging/broker/broker_grpc_server_publish.go b/weed/messaging/broker/broker_grpc_server_publish.go index 61e53b433..573706c06 100644 --- a/weed/messaging/broker/broker_grpc_server_publish.go +++ b/weed/messaging/broker/broker_grpc_server_publish.go @@ -1,11 +1,15 @@ package broker import ( + "crypto/md5" + "fmt" "io" "github.com/golang/protobuf/proto" + "github.com/chrislusf/seaweedfs/weed/filer2" "github.com/chrislusf/seaweedfs/weed/glog" + "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "github.com/chrislusf/seaweedfs/weed/pb/messaging_pb" ) @@ -44,9 +48,19 @@ func (broker *MessageBroker) Publish(stream messaging_pb.SeaweedMessaging_Publis Topic: in.Init.Topic, Partition: in.Init.Partition, } + + tpDir := fmt.Sprintf("%s/%s/%s", filer2.TopicsDir, tp.Namespace, tp.Topic) + md5File := fmt.Sprintf("p%02d.md5", tp.Partition) + // println("chan data stored under", tpDir, "as", md5File) + + if exists, err := filer_pb.Exists(broker, tpDir, md5File, false); err == nil && exists { + return fmt.Errorf("channel is already closed") + } + tl := broker.topicLocks.RequestLock(tp, topicConfig, true) defer broker.topicLocks.ReleaseLock(tp, true) + md5hash := md5.New() // process each message for { // println("recv") @@ -78,8 +92,16 @@ func (broker *MessageBroker) Publish(stream messaging_pb.SeaweedMessaging_Publis break } + md5hash.Write(in.Data.Value) + + } + + if err := broker.appendToFile(tpDir+"/"+md5File, topicConfig, md5hash.Sum(nil)); err != nil { + glog.V(0).Infof("err writing %s: %v", md5File, err) } + // fmt.Printf("received md5 %X\n", md5hash.Sum(nil)) + // send the close ack // println("server send ack closing") if err := stream.Send(&messaging_pb.PublishResponse{IsClosed: true}); err != nil { diff --git a/weed/messaging/broker/broker_grpc_server_subscribe.go b/weed/messaging/broker/broker_grpc_server_subscribe.go index f8fd16a14..86ee6923d 100644 --- a/weed/messaging/broker/broker_grpc_server_subscribe.go +++ b/weed/messaging/broker/broker_grpc_server_subscribe.go @@ -57,6 +57,7 @@ func (broker *MessageBroker) Subscribe(stream messaging_pb.SeaweedMessaging_Subs lastReadTime = time.Unix(0, in.Init.TimestampNs) case messaging_pb.SubscriberMessage_InitMessage_LATEST: case messaging_pb.SubscriberMessage_InitMessage_EARLIEST: + lastReadTime = time.Unix(0, 0) } var processedTsNs int64 diff --git a/weed/messaging/broker/topic_lock.go b/weed/messaging/broker/topic_lock.go index f3a66a2f5..4c4803275 100644 --- a/weed/messaging/broker/topic_lock.go +++ b/weed/messaging/broker/topic_lock.go @@ -17,7 +17,7 @@ type TopicPartition struct { Partition int32 } const ( - TopicPartitionFmt = "%s/%s_%2d" + TopicPartitionFmt = "%s/%s_%02d" ) func (tp *TopicPartition) String() string { return fmt.Sprintf(TopicPartitionFmt, tp.Namespace, tp.Topic, tp.Partition) @@ -106,6 +106,7 @@ func (tl *TopicLocks) ReleaseLock(partition TopicPartition, isPublisher bool) { } if lock.subscriberCount <= 0 && lock.publisherCount <= 0 { delete(tl.locks, partition) + lock.logBuffer.Shutdown() } } diff --git a/weed/messaging/msgclient/sub_chan.go b/weed/messaging/msgclient/sub_chan.go index aae5c0c71..ed25a850c 100644 --- a/weed/messaging/msgclient/sub_chan.go +++ b/weed/messaging/msgclient/sub_chan.go @@ -55,8 +55,8 @@ func (mc *MessagingClient) NewSubChannel(chanName string) (*SubChannel, error) { close(t.ch) return } - t.md5hash.Write(resp.Data.Value) t.ch <- resp.Data.Value + t.md5hash.Write(resp.Data.Value) } }() diff --git a/weed/util/log_buffer/log_buffer.go b/weed/util/log_buffer/log_buffer.go index 6ba7f3737..67c44dc57 100644 --- a/weed/util/log_buffer/log_buffer.go +++ b/weed/util/log_buffer/log_buffer.go @@ -98,13 +98,14 @@ func (m *LogBuffer) AddToBuffer(partitionKey, data []byte) { } func (m *LogBuffer) Shutdown() { + m.Lock() + defer m.Unlock() + if m.isStopping { return } m.isStopping = true - m.Lock() toFlush := m.copyToFlush() - m.Unlock() m.flushChan <- toFlush close(m.flushChan) } @@ -123,10 +124,14 @@ func (m *LogBuffer) loopInterval() { for !m.isStopping { time.Sleep(m.flushInterval) m.Lock() + if m.isStopping { + m.Unlock() + return + } // println("loop interval") toFlush := m.copyToFlush() - m.Unlock() m.flushChan <- toFlush + m.Unlock() } } From 39e72fb23ca0d23cc563da50746c7a717bfa01e6 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sun, 10 May 2020 03:50:30 -0700 Subject: [PATCH 06/24] go fmt --- weed/command/benchmark.go | 2 +- weed/command/filer_copy.go | 2 +- weed/filer2/reader_at.go | 2 +- weed/filer2/stream.go | 2 +- weed/messaging/broker/broker_grpc_server_subscribe.go | 3 +-- weed/messaging/broker/consistent_distribution.go | 4 ++-- weed/messaging/broker/consistent_distribution_test.go | 4 ++-- weed/messaging/broker/topic_lock.go | 2 ++ weed/messaging/msgclient/client.go | 1 - weed/messaging/msgclient/publisher.go | 1 + weed/messaging/msgclient/subscriber.go | 2 +- weed/operation/submit.go | 2 +- weed/util/log_buffer/sealed_buffer.go | 2 +- 13 files changed, 15 insertions(+), 14 deletions(-) diff --git a/weed/command/benchmark.go b/weed/command/benchmark.go index 5b2e31622..ee21914b3 100644 --- a/weed/command/benchmark.go +++ b/weed/command/benchmark.go @@ -232,7 +232,7 @@ func writeFiles(idChan chan int, fileIdLineChan chan string, s *stat) { Reader: &FakeReader{id: uint64(id), size: fileSize, random: random}, FileSize: fileSize, MimeType: "image/bench", // prevent gzip benchmark content - Fsync: *b.fsync, + Fsync: *b.fsync, } ar := &operation.VolumeAssignRequest{ Count: 1, diff --git a/weed/command/filer_copy.go b/weed/command/filer_copy.go index 322ab20d5..2d6ba94d6 100644 --- a/weed/command/filer_copy.go +++ b/weed/command/filer_copy.go @@ -428,7 +428,7 @@ func (worker *FileCopyWorker) uploadFileInChunks(task FileCopyTask, f *os.File, uploadError = fmt.Errorf("upload %v to %s result: %v\n", fileName, targetUrl, uploadResult.Error) return } - chunksChan <- uploadResult.ToPbFileChunk(assignResult.FileId, i * chunkSize) + chunksChan <- uploadResult.ToPbFileChunk(assignResult.FileId, i*chunkSize) fmt.Printf("uploaded %s-%d to %s [%d,%d)\n", fileName, i+1, targetUrl, i*chunkSize, i*chunkSize+int64(uploadResult.Size)) }(i) diff --git a/weed/filer2/reader_at.go b/weed/filer2/reader_at.go index a9772da5b..53fff7672 100644 --- a/weed/filer2/reader_at.go +++ b/weed/filer2/reader_at.go @@ -59,7 +59,7 @@ func LookupFn(filerClient filer_pb.FilerClient) LookupFileIdFunctionType { func NewChunkReaderAtFromClient(filerClient filer_pb.FilerClient, chunkViews []*ChunkView, chunkCache *chunk_cache.ChunkCache) *ChunkReadAt { return &ChunkReadAt{ - chunkViews: chunkViews, + chunkViews: chunkViews, lookupFileId: LookupFn(filerClient), bufferOffset: -1, chunkCache: chunkCache, diff --git a/weed/filer2/stream.go b/weed/filer2/stream.go index 4c8213b07..033a8dd13 100644 --- a/weed/filer2/stream.go +++ b/weed/filer2/stream.go @@ -103,7 +103,7 @@ func NewChunkStreamReader(filerClient filer_pb.FilerClient, chunks []*filer_pb.F chunkViews := ViewFromChunks(chunks, 0, math.MaxInt32) return &ChunkStreamReader{ - chunkViews: chunkViews, + chunkViews: chunkViews, lookupFileId: LookupFn(filerClient), } } diff --git a/weed/messaging/broker/broker_grpc_server_subscribe.go b/weed/messaging/broker/broker_grpc_server_subscribe.go index 86ee6923d..76cbdef24 100644 --- a/weed/messaging/broker/broker_grpc_server_subscribe.go +++ b/weed/messaging/broker/broker_grpc_server_subscribe.go @@ -37,8 +37,7 @@ func (broker *MessageBroker) Subscribe(stream messaging_pb.SeaweedMessaging_Subs // IsTransient: true, } - if err = stream.Send(&messaging_pb.BrokerMessage{ - }); err != nil { + if err = stream.Send(&messaging_pb.BrokerMessage{}); err != nil { return err } diff --git a/weed/messaging/broker/consistent_distribution.go b/weed/messaging/broker/consistent_distribution.go index dd7d34f86..465a2a8f2 100644 --- a/weed/messaging/broker/consistent_distribution.go +++ b/weed/messaging/broker/consistent_distribution.go @@ -1,8 +1,8 @@ package broker import ( - "github.com/cespare/xxhash" "github.com/buraksezer/consistent" + "github.com/cespare/xxhash" ) type Member string @@ -35,4 +35,4 @@ func PickMember(members []string, key []byte) string { m := c.LocateKey(key) return m.String() -} \ No newline at end of file +} diff --git a/weed/messaging/broker/consistent_distribution_test.go b/weed/messaging/broker/consistent_distribution_test.go index 192516092..f58fe4e0e 100644 --- a/weed/messaging/broker/consistent_distribution_test.go +++ b/weed/messaging/broker/consistent_distribution_test.go @@ -18,7 +18,7 @@ func TestPickMember(t *testing.T) { total := 1000 distribution := make(map[string]int) - for i:=0;i", m) @@ -29,4 +29,4 @@ func TestPickMember(t *testing.T) { fmt.Printf("member: %s, key count: %d load=%.2f\n", member, count, float64(count*100)/float64(total/len(servers))) } -} \ No newline at end of file +} diff --git a/weed/messaging/broker/topic_lock.go b/weed/messaging/broker/topic_lock.go index 4c4803275..9ae446df3 100644 --- a/weed/messaging/broker/topic_lock.go +++ b/weed/messaging/broker/topic_lock.go @@ -16,9 +16,11 @@ type TopicPartition struct { Topic string Partition int32 } + const ( TopicPartitionFmt = "%s/%s_%02d" ) + func (tp *TopicPartition) String() string { return fmt.Sprintf(TopicPartitionFmt, tp.Namespace, tp.Topic, tp.Partition) } diff --git a/weed/messaging/msgclient/client.go b/weed/messaging/msgclient/client.go index f4e11232e..4d7ef2b8e 100644 --- a/weed/messaging/msgclient/client.go +++ b/weed/messaging/msgclient/client.go @@ -28,7 +28,6 @@ func NewMessagingClient(bootstrapBrokers ...string) *MessagingClient { } } - func (mc *MessagingClient) findBroker(tp broker.TopicPartition) (*grpc.ClientConn, error) { for _, broker := range mc.bootstrapBrokers { diff --git a/weed/messaging/msgclient/publisher.go b/weed/messaging/msgclient/publisher.go index 08f1d278a..ebb6d3f2a 100644 --- a/weed/messaging/msgclient/publisher.go +++ b/weed/messaging/msgclient/publisher.go @@ -16,6 +16,7 @@ type Publisher struct { messageCount uint64 publisherId string } + /* func (mc *MessagingClient) NewPublisher(publisherId, namespace, topic string) (*Publisher, error) { // read topic configuration diff --git a/weed/messaging/msgclient/subscriber.go b/weed/messaging/msgclient/subscriber.go index d3066d6ef..efbfa0337 100644 --- a/weed/messaging/msgclient/subscriber.go +++ b/weed/messaging/msgclient/subscriber.go @@ -5,8 +5,8 @@ import ( "io" "time" - "google.golang.org/grpc" "github.com/chrislusf/seaweedfs/weed/pb/messaging_pb" + "google.golang.org/grpc" ) type Subscriber struct { diff --git a/weed/operation/submit.go b/weed/operation/submit.go index 7cf5b9645..e8bec382a 100644 --- a/weed/operation/submit.go +++ b/weed/operation/submit.go @@ -27,7 +27,7 @@ type FilePart struct { Ttl string Server string //this comes from assign result Fid string //this comes from assign result, but customizable - Fsync bool + Fsync bool } type SubmitResult struct { diff --git a/weed/util/log_buffer/sealed_buffer.go b/weed/util/log_buffer/sealed_buffer.go index cade45dd1..d133cf8d3 100644 --- a/weed/util/log_buffer/sealed_buffer.go +++ b/weed/util/log_buffer/sealed_buffer.go @@ -59,4 +59,4 @@ func (mb *MemBuffer) locateByTs(lastReadTime time.Time) (pos int) { func (mb *MemBuffer) String() string { return fmt.Sprintf("[%v,%v] bytes:%d", mb.startTime, mb.stopTime, mb.size) -} \ No newline at end of file +} From d5a8297a1cfeaa296cb160a6195ca9e14230048c Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sun, 10 May 2020 23:45:48 -0700 Subject: [PATCH 07/24] able to connect publisher --- weed/messaging/msgclient/publisher.go | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/weed/messaging/msgclient/publisher.go b/weed/messaging/msgclient/publisher.go index ebb6d3f2a..b0fc5afbf 100644 --- a/weed/messaging/msgclient/publisher.go +++ b/weed/messaging/msgclient/publisher.go @@ -17,7 +17,6 @@ type Publisher struct { publisherId string } -/* func (mc *MessagingClient) NewPublisher(publisherId, namespace, topic string) (*Publisher, error) { // read topic configuration topicConfiguration := &messaging_pb.TopicConfiguration{ @@ -25,11 +24,16 @@ func (mc *MessagingClient) NewPublisher(publisherId, namespace, topic string) (* } publishClients := make([]messaging_pb.SeaweedMessaging_PublishClient, topicConfiguration.PartitionCount) for i := 0; i < int(topicConfiguration.PartitionCount); i++ { - client, err := setupPublisherClient(broker.TopicPartition{ + tp := broker.TopicPartition{ Namespace: namespace, Topic: topic, Partition: int32(i), - }) + } + grpcClientConn, err := mc.findBroker(tp) + if err != nil { + return nil, err + } + client, err := setupPublisherClient(grpcClientConn, tp) if err != nil { return nil, err } @@ -40,7 +44,6 @@ func (mc *MessagingClient) NewPublisher(publisherId, namespace, topic string) (* topicConfiguration: topicConfiguration, }, nil } -*/ func setupPublisherClient(grpcConnection *grpc.ClientConn, tp broker.TopicPartition) (messaging_pb.SeaweedMessaging_PublishClient, error) { @@ -113,9 +116,3 @@ func (p *Publisher) Publish(m *messaging_pb.Message) error { Data: m, }) } - -func (p *Publisher) Shutdown() { - for _, client := range p.publishClients { - client.CloseSend() - } -} From 4b7fa31468eb1b8e0af53543a7e760c517faf227 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Mon, 11 May 2020 01:53:54 -0700 Subject: [PATCH 08/24] ensure montonically increasing tsNs --- weed/util/log_buffer/log_buffer.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/weed/util/log_buffer/log_buffer.go b/weed/util/log_buffer/log_buffer.go index 67c44dc57..b02c45b52 100644 --- a/weed/util/log_buffer/log_buffer.go +++ b/weed/util/log_buffer/log_buffer.go @@ -34,6 +34,7 @@ type LogBuffer struct { notifyFn func() isStopping bool flushChan chan *dataToFlush + lastTsNs int64 sync.RWMutex } @@ -64,8 +65,15 @@ func (m *LogBuffer) AddToBuffer(partitionKey, data []byte) { // need to put the timestamp inside the lock ts := time.Now() + tsNs := ts.UnixNano() + if m.lastTsNs >= tsNs { + // this is unlikely to happen, but just in case + tsNs = m.lastTsNs + 1 + ts = time.Unix(0, tsNs) + } + m.lastTsNs = tsNs logEntry := &filer_pb.LogEntry{ - TsNs: ts.UnixNano(), + TsNs: tsNs, PartitionKeyHash: util.HashToInt32(partitionKey), Data: data, } From 2f243f5b0b7c037eb13ae14f58bd1f8608fbc4d0 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Tue, 12 May 2020 08:48:00 -0700 Subject: [PATCH 09/24] refactor --- .../broker/broker_grpc_server_publish.go | 4 +- .../broker/broker_grpc_server_subscribe.go | 4 +- weed/messaging/broker/broker_server.go | 6 +-- .../{topic_lock.go => topic_manager.go} | 52 +++++++++---------- 4 files changed, 33 insertions(+), 33 deletions(-) rename weed/messaging/broker/{topic_lock.go => topic_manager.go} (57%) diff --git a/weed/messaging/broker/broker_grpc_server_publish.go b/weed/messaging/broker/broker_grpc_server_publish.go index 573706c06..dc11061af 100644 --- a/weed/messaging/broker/broker_grpc_server_publish.go +++ b/weed/messaging/broker/broker_grpc_server_publish.go @@ -57,8 +57,8 @@ func (broker *MessageBroker) Publish(stream messaging_pb.SeaweedMessaging_Publis return fmt.Errorf("channel is already closed") } - tl := broker.topicLocks.RequestLock(tp, topicConfig, true) - defer broker.topicLocks.ReleaseLock(tp, true) + tl := broker.topicManager.RequestLock(tp, topicConfig, true) + defer broker.topicManager.ReleaseLock(tp, true) md5hash := md5.New() // process each message diff --git a/weed/messaging/broker/broker_grpc_server_subscribe.go b/weed/messaging/broker/broker_grpc_server_subscribe.go index 76cbdef24..e7cbb6441 100644 --- a/weed/messaging/broker/broker_grpc_server_subscribe.go +++ b/weed/messaging/broker/broker_grpc_server_subscribe.go @@ -47,8 +47,8 @@ func (broker *MessageBroker) Subscribe(stream messaging_pb.SeaweedMessaging_Subs Topic: in.Init.Topic, Partition: in.Init.Partition, } - lock := broker.topicLocks.RequestLock(tp, topicConfig, false) - defer broker.topicLocks.ReleaseLock(tp, false) + lock := broker.topicManager.RequestLock(tp, topicConfig, false) + defer broker.topicManager.ReleaseLock(tp, false) lastReadTime := time.Now() switch in.Init.StartPosition { diff --git a/weed/messaging/broker/broker_server.go b/weed/messaging/broker/broker_server.go index e6ff2cf00..0c04d2841 100644 --- a/weed/messaging/broker/broker_server.go +++ b/weed/messaging/broker/broker_server.go @@ -24,7 +24,7 @@ type MessageBrokerOption struct { type MessageBroker struct { option *MessageBrokerOption grpcDialOption grpc.DialOption - topicLocks *TopicLocks + topicManager *TopicManager } func NewMessageBroker(option *MessageBrokerOption, grpcDialOption grpc.DialOption) (messageBroker *MessageBroker, err error) { @@ -34,7 +34,7 @@ func NewMessageBroker(option *MessageBrokerOption, grpcDialOption grpc.DialOptio grpcDialOption: grpcDialOption, } - messageBroker.topicLocks = NewTopicLocks(messageBroker) + messageBroker.topicManager = NewTopicManager(messageBroker) messageBroker.checkFilers() @@ -58,7 +58,7 @@ func (broker *MessageBroker) keepConnectedToOneFiler() { Name: broker.option.Ip, GrpcPort: uint32(broker.option.Port), } - for _, tp := range broker.topicLocks.ListTopicPartitions() { + for _, tp := range broker.topicManager.ListTopicPartitions() { initRequest.Resources = append(initRequest.Resources, tp.String()) } if err := stream.Send(&filer_pb.KeepConnectedRequest{ diff --git a/weed/messaging/broker/topic_lock.go b/weed/messaging/broker/topic_manager.go similarity index 57% rename from weed/messaging/broker/topic_lock.go rename to weed/messaging/broker/topic_manager.go index 9ae446df3..21594dea5 100644 --- a/weed/messaging/broker/topic_lock.go +++ b/weed/messaging/broker/topic_manager.go @@ -25,7 +25,7 @@ func (tp *TopicPartition) String() string { return fmt.Sprintf(TopicPartitionFmt, tp.Namespace, tp.Topic, tp.Partition) } -type TopicLock struct { +type TopicControl struct { sync.Mutex cond *sync.Cond subscriberCount int @@ -33,20 +33,20 @@ type TopicLock struct { logBuffer *log_buffer.LogBuffer } -type TopicLocks struct { +type TopicManager struct { sync.Mutex - locks map[TopicPartition]*TopicLock - broker *MessageBroker + topicControls map[TopicPartition]*TopicControl + broker *MessageBroker } -func NewTopicLocks(messageBroker *MessageBroker) *TopicLocks { - return &TopicLocks{ - locks: make(map[TopicPartition]*TopicLock), - broker: messageBroker, +func NewTopicManager(messageBroker *MessageBroker) *TopicManager { + return &TopicManager{ + topicControls: make(map[TopicPartition]*TopicControl), + broker: messageBroker, } } -func (locks *TopicLocks) buildLogBuffer(tl *TopicLock, tp TopicPartition, topicConfig *messaging_pb.TopicConfiguration) *log_buffer.LogBuffer { +func (tm *TopicManager) buildLogBuffer(tl *TopicControl, tp TopicPartition, topicConfig *messaging_pb.TopicConfiguration) *log_buffer.LogBuffer { flushFn := func(startTime, stopTime time.Time, buf []byte) { @@ -63,7 +63,7 @@ func (locks *TopicLocks) buildLogBuffer(tl *TopicLock, tp TopicPartition, topicC tp.Partition, ) - if err := locks.broker.appendToFile(targetFile, topicConfig, buf); err != nil { + if err := tm.broker.appendToFile(targetFile, topicConfig, buf); err != nil { glog.V(0).Infof("log write failed %s: %v", targetFile, err) } } @@ -74,16 +74,16 @@ func (locks *TopicLocks) buildLogBuffer(tl *TopicLock, tp TopicPartition, topicC return logBuffer } -func (tl *TopicLocks) RequestLock(partition TopicPartition, topicConfig *messaging_pb.TopicConfiguration, isPublisher bool) *TopicLock { - tl.Lock() - defer tl.Unlock() +func (tm *TopicManager) RequestLock(partition TopicPartition, topicConfig *messaging_pb.TopicConfiguration, isPublisher bool) *TopicControl { + tm.Lock() + defer tm.Unlock() - lock, found := tl.locks[partition] + lock, found := tm.topicControls[partition] if !found { - lock = &TopicLock{} + lock = &TopicControl{} lock.cond = sync.NewCond(&lock.Mutex) - tl.locks[partition] = lock - lock.logBuffer = tl.buildLogBuffer(lock, partition, topicConfig) + tm.topicControls[partition] = lock + lock.logBuffer = tm.buildLogBuffer(lock, partition, topicConfig) } if isPublisher { lock.publisherCount++ @@ -93,11 +93,11 @@ func (tl *TopicLocks) RequestLock(partition TopicPartition, topicConfig *messagi return lock } -func (tl *TopicLocks) ReleaseLock(partition TopicPartition, isPublisher bool) { - tl.Lock() - defer tl.Unlock() +func (tm *TopicManager) ReleaseLock(partition TopicPartition, isPublisher bool) { + tm.Lock() + defer tm.Unlock() - lock, found := tl.locks[partition] + lock, found := tm.topicControls[partition] if !found { return } @@ -107,16 +107,16 @@ func (tl *TopicLocks) ReleaseLock(partition TopicPartition, isPublisher bool) { lock.subscriberCount-- } if lock.subscriberCount <= 0 && lock.publisherCount <= 0 { - delete(tl.locks, partition) + delete(tm.topicControls, partition) lock.logBuffer.Shutdown() } } -func (tl *TopicLocks) ListTopicPartitions() (tps []TopicPartition) { - tl.Lock() - defer tl.Unlock() +func (tm *TopicManager) ListTopicPartitions() (tps []TopicPartition) { + tm.Lock() + defer tm.Unlock() - for k := range tl.locks { + for k := range tm.topicControls { tps = append(tps, k) } return From a7959c1c488f5662dc2f867e0dbeeebf3899bbe3 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Tue, 12 May 2020 21:26:02 -0700 Subject: [PATCH 10/24] multiple subscriber with same subscriberId shares the topic manager rename topicControl to topicCursor --- .../broker/broker_grpc_server_subscribe.go | 5 +- weed/messaging/broker/subscription.go | 82 +++++++++++++++++++ weed/messaging/broker/topic_manager.go | 17 ++-- weed/messaging/msgclient/sub_chan.go | 4 +- 4 files changed, 95 insertions(+), 13 deletions(-) create mode 100644 weed/messaging/broker/subscription.go diff --git a/weed/messaging/broker/broker_grpc_server_subscribe.go b/weed/messaging/broker/broker_grpc_server_subscribe.go index e7cbb6441..eb6946e81 100644 --- a/weed/messaging/broker/broker_grpc_server_subscribe.go +++ b/weed/messaging/broker/broker_grpc_server_subscribe.go @@ -48,6 +48,7 @@ func (broker *MessageBroker) Subscribe(stream messaging_pb.SeaweedMessaging_Subs Partition: in.Init.Partition, } lock := broker.topicManager.RequestLock(tp, topicConfig, false) + subscription := lock.subscriptions.AddSubscription(subscriberId) defer broker.topicManager.ReleaseLock(tp, false) lastReadTime := time.Now() @@ -102,9 +103,7 @@ func (broker *MessageBroker) Subscribe(stream messaging_pb.SeaweedMessaging_Subs } err = lock.logBuffer.LoopProcessLogData(lastReadTime, func() bool { - lock.Mutex.Lock() - lock.cond.Wait() - lock.Mutex.Unlock() + subscription.Wait() return true }, eachLogEntryFn) diff --git a/weed/messaging/broker/subscription.go b/weed/messaging/broker/subscription.go new file mode 100644 index 000000000..f74b0c546 --- /dev/null +++ b/weed/messaging/broker/subscription.go @@ -0,0 +1,82 @@ +package broker + +import ( + "sync" +) + +type TopicPartitionSubscription struct { + sync.Mutex + name string + lastReadTsNs int64 + cond *sync.Cond +} + +func NewTopicPartitionSubscription(name string) *TopicPartitionSubscription { + t := &TopicPartitionSubscription{ + name: name, + } + t.cond = sync.NewCond(t) + return t +} + +func (s *TopicPartitionSubscription) Wait() { + s.Mutex.Lock() + s.cond.Wait() + s.Mutex.Unlock() +} + +func (s *TopicPartitionSubscription) NotifyOne() { + // notify one waiting goroutine + s.cond.Signal() +} + +type TopicPartitionSubscriptions struct { + sync.Mutex + cond *sync.Cond + subscriptions map[string]*TopicPartitionSubscription + subscriptionsLock sync.RWMutex +} + +func NewTopicPartitionSubscriptions() *TopicPartitionSubscriptions { + m := &TopicPartitionSubscriptions{ + subscriptions: make(map[string]*TopicPartitionSubscription), + } + m.cond = sync.NewCond(m) + return m +} + +func (m *TopicPartitionSubscriptions) AddSubscription(subscription string) *TopicPartitionSubscription { + m.subscriptionsLock.Lock() + defer m.subscriptionsLock.Unlock() + + if s, found := m.subscriptions[subscription]; found { + return s + } + + s := NewTopicPartitionSubscription(subscription) + m.subscriptions[subscription] = s + + return s + +} + +func (m *TopicPartitionSubscriptions) NotifyAll() { + + m.subscriptionsLock.RLock() + defer m.subscriptionsLock.RUnlock() + + for name, tps := range m.subscriptions { + println("notifying", name) + tps.NotifyOne() + } + +} + +func (m *TopicPartitionSubscriptions) Wait() { + m.Mutex.Lock() + m.cond.Wait() + for _, tps := range m.subscriptions { + tps.NotifyOne() + } + m.Mutex.Unlock() +} diff --git a/weed/messaging/broker/topic_manager.go b/weed/messaging/broker/topic_manager.go index 21594dea5..e5cebb42d 100644 --- a/weed/messaging/broker/topic_manager.go +++ b/weed/messaging/broker/topic_manager.go @@ -25,28 +25,29 @@ func (tp *TopicPartition) String() string { return fmt.Sprintf(TopicPartitionFmt, tp.Namespace, tp.Topic, tp.Partition) } -type TopicControl struct { +type TopicCursor struct { sync.Mutex cond *sync.Cond subscriberCount int publisherCount int logBuffer *log_buffer.LogBuffer + subscriptions *TopicPartitionSubscriptions } type TopicManager struct { sync.Mutex - topicControls map[TopicPartition]*TopicControl + topicControls map[TopicPartition]*TopicCursor broker *MessageBroker } func NewTopicManager(messageBroker *MessageBroker) *TopicManager { return &TopicManager{ - topicControls: make(map[TopicPartition]*TopicControl), + topicControls: make(map[TopicPartition]*TopicCursor), broker: messageBroker, } } -func (tm *TopicManager) buildLogBuffer(tl *TopicControl, tp TopicPartition, topicConfig *messaging_pb.TopicConfiguration) *log_buffer.LogBuffer { +func (tm *TopicManager) buildLogBuffer(tl *TopicCursor, tp TopicPartition, topicConfig *messaging_pb.TopicConfiguration) *log_buffer.LogBuffer { flushFn := func(startTime, stopTime time.Time, buf []byte) { @@ -68,21 +69,21 @@ func (tm *TopicManager) buildLogBuffer(tl *TopicControl, tp TopicPartition, topi } } logBuffer := log_buffer.NewLogBuffer(time.Minute, flushFn, func() { - tl.cond.Broadcast() + tl.subscriptions.NotifyAll() }) return logBuffer } -func (tm *TopicManager) RequestLock(partition TopicPartition, topicConfig *messaging_pb.TopicConfiguration, isPublisher bool) *TopicControl { +func (tm *TopicManager) RequestLock(partition TopicPartition, topicConfig *messaging_pb.TopicConfiguration, isPublisher bool) *TopicCursor { tm.Lock() defer tm.Unlock() lock, found := tm.topicControls[partition] if !found { - lock = &TopicControl{} - lock.cond = sync.NewCond(&lock.Mutex) + lock = &TopicCursor{} tm.topicControls[partition] = lock + lock.subscriptions = NewTopicPartitionSubscriptions() lock.logBuffer = tm.buildLogBuffer(lock, partition, topicConfig) } if isPublisher { diff --git a/weed/messaging/msgclient/sub_chan.go b/weed/messaging/msgclient/sub_chan.go index ed25a850c..5465c5913 100644 --- a/weed/messaging/msgclient/sub_chan.go +++ b/weed/messaging/msgclient/sub_chan.go @@ -17,7 +17,7 @@ type SubChannel struct { md5hash hash.Hash } -func (mc *MessagingClient) NewSubChannel(chanName string) (*SubChannel, error) { +func (mc *MessagingClient) NewSubChannel(subscriberId, chanName string) (*SubChannel, error) { tp := broker.TopicPartition{ Namespace: "chan", Topic: chanName, @@ -27,7 +27,7 @@ func (mc *MessagingClient) NewSubChannel(chanName string) (*SubChannel, error) { if err != nil { return nil, err } - sc, err := setupSubscriberClient(grpcConnection, "", "chan", chanName, 0, time.Unix(0, 0)) + sc, err := setupSubscriberClient(grpcConnection, subscriberId, "chan", chanName, 0, time.Unix(0, 0)) if err != nil { return nil, err } From 25257acd51085c3be360f6f516e7019b79c1b813 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Tue, 12 May 2020 21:26:49 -0700 Subject: [PATCH 11/24] rename --- weed/messaging/broker/topic_manager.go | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/weed/messaging/broker/topic_manager.go b/weed/messaging/broker/topic_manager.go index e5cebb42d..7cc8601eb 100644 --- a/weed/messaging/broker/topic_manager.go +++ b/weed/messaging/broker/topic_manager.go @@ -36,14 +36,14 @@ type TopicCursor struct { type TopicManager struct { sync.Mutex - topicControls map[TopicPartition]*TopicCursor - broker *MessageBroker + topicCursors map[TopicPartition]*TopicCursor + broker *MessageBroker } func NewTopicManager(messageBroker *MessageBroker) *TopicManager { return &TopicManager{ - topicControls: make(map[TopicPartition]*TopicCursor), - broker: messageBroker, + topicCursors: make(map[TopicPartition]*TopicCursor), + broker: messageBroker, } } @@ -79,10 +79,10 @@ func (tm *TopicManager) RequestLock(partition TopicPartition, topicConfig *messa tm.Lock() defer tm.Unlock() - lock, found := tm.topicControls[partition] + lock, found := tm.topicCursors[partition] if !found { lock = &TopicCursor{} - tm.topicControls[partition] = lock + tm.topicCursors[partition] = lock lock.subscriptions = NewTopicPartitionSubscriptions() lock.logBuffer = tm.buildLogBuffer(lock, partition, topicConfig) } @@ -98,7 +98,7 @@ func (tm *TopicManager) ReleaseLock(partition TopicPartition, isPublisher bool) tm.Lock() defer tm.Unlock() - lock, found := tm.topicControls[partition] + lock, found := tm.topicCursors[partition] if !found { return } @@ -108,7 +108,7 @@ func (tm *TopicManager) ReleaseLock(partition TopicPartition, isPublisher bool) lock.subscriberCount-- } if lock.subscriberCount <= 0 && lock.publisherCount <= 0 { - delete(tm.topicControls, partition) + delete(tm.topicCursors, partition) lock.logBuffer.Shutdown() } } @@ -117,7 +117,7 @@ func (tm *TopicManager) ListTopicPartitions() (tps []TopicPartition) { tm.Lock() defer tm.Unlock() - for k := range tm.topicControls { + for k := range tm.topicCursors { tps = append(tps, k) } return From ca4017dd87ce23198e4589b7cf631ad6276ea564 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Tue, 12 May 2020 22:55:55 -0700 Subject: [PATCH 12/24] rename --- weed/messaging/broker/topic_manager.go | 28 +++++++++++++------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/weed/messaging/broker/topic_manager.go b/weed/messaging/broker/topic_manager.go index 7cc8601eb..e9f8903e8 100644 --- a/weed/messaging/broker/topic_manager.go +++ b/weed/messaging/broker/topic_manager.go @@ -25,7 +25,7 @@ func (tp *TopicPartition) String() string { return fmt.Sprintf(TopicPartitionFmt, tp.Namespace, tp.Topic, tp.Partition) } -type TopicCursor struct { +type TopicControl struct { sync.Mutex cond *sync.Cond subscriberCount int @@ -36,18 +36,18 @@ type TopicCursor struct { type TopicManager struct { sync.Mutex - topicCursors map[TopicPartition]*TopicCursor + topicCursors map[TopicPartition]*TopicControl broker *MessageBroker } func NewTopicManager(messageBroker *MessageBroker) *TopicManager { return &TopicManager{ - topicCursors: make(map[TopicPartition]*TopicCursor), + topicCursors: make(map[TopicPartition]*TopicControl), broker: messageBroker, } } -func (tm *TopicManager) buildLogBuffer(tl *TopicCursor, tp TopicPartition, topicConfig *messaging_pb.TopicConfiguration) *log_buffer.LogBuffer { +func (tm *TopicManager) buildLogBuffer(tc *TopicControl, tp TopicPartition, topicConfig *messaging_pb.TopicConfiguration) *log_buffer.LogBuffer { flushFn := func(startTime, stopTime time.Time, buf []byte) { @@ -69,29 +69,29 @@ func (tm *TopicManager) buildLogBuffer(tl *TopicCursor, tp TopicPartition, topic } } logBuffer := log_buffer.NewLogBuffer(time.Minute, flushFn, func() { - tl.subscriptions.NotifyAll() + tc.subscriptions.NotifyAll() }) return logBuffer } -func (tm *TopicManager) RequestLock(partition TopicPartition, topicConfig *messaging_pb.TopicConfiguration, isPublisher bool) *TopicCursor { +func (tm *TopicManager) RequestLock(partition TopicPartition, topicConfig *messaging_pb.TopicConfiguration, isPublisher bool) *TopicControl { tm.Lock() defer tm.Unlock() - lock, found := tm.topicCursors[partition] + tc, found := tm.topicCursors[partition] if !found { - lock = &TopicCursor{} - tm.topicCursors[partition] = lock - lock.subscriptions = NewTopicPartitionSubscriptions() - lock.logBuffer = tm.buildLogBuffer(lock, partition, topicConfig) + tc = &TopicControl{} + tm.topicCursors[partition] = tc + tc.subscriptions = NewTopicPartitionSubscriptions() + tc.logBuffer = tm.buildLogBuffer(tc, partition, topicConfig) } if isPublisher { - lock.publisherCount++ + tc.publisherCount++ } else { - lock.subscriberCount++ + tc.subscriberCount++ } - return lock + return tc } func (tm *TopicManager) ReleaseLock(partition TopicPartition, isPublisher bool) { From e02a8c67daade10a305834e1c19c0c73905ce1c6 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Fri, 15 May 2020 21:38:42 -0700 Subject: [PATCH 13/24] revert to one subscriber one thread --- .../broker/broker_grpc_server_subscribe.go | 5 +- weed/messaging/broker/subscription.go | 82 ------------------- weed/messaging/broker/topic_manager.go | 25 +++--- 3 files changed, 15 insertions(+), 97 deletions(-) delete mode 100644 weed/messaging/broker/subscription.go diff --git a/weed/messaging/broker/broker_grpc_server_subscribe.go b/weed/messaging/broker/broker_grpc_server_subscribe.go index eb6946e81..e7cbb6441 100644 --- a/weed/messaging/broker/broker_grpc_server_subscribe.go +++ b/weed/messaging/broker/broker_grpc_server_subscribe.go @@ -48,7 +48,6 @@ func (broker *MessageBroker) Subscribe(stream messaging_pb.SeaweedMessaging_Subs Partition: in.Init.Partition, } lock := broker.topicManager.RequestLock(tp, topicConfig, false) - subscription := lock.subscriptions.AddSubscription(subscriberId) defer broker.topicManager.ReleaseLock(tp, false) lastReadTime := time.Now() @@ -103,7 +102,9 @@ func (broker *MessageBroker) Subscribe(stream messaging_pb.SeaweedMessaging_Subs } err = lock.logBuffer.LoopProcessLogData(lastReadTime, func() bool { - subscription.Wait() + lock.Mutex.Lock() + lock.cond.Wait() + lock.Mutex.Unlock() return true }, eachLogEntryFn) diff --git a/weed/messaging/broker/subscription.go b/weed/messaging/broker/subscription.go deleted file mode 100644 index f74b0c546..000000000 --- a/weed/messaging/broker/subscription.go +++ /dev/null @@ -1,82 +0,0 @@ -package broker - -import ( - "sync" -) - -type TopicPartitionSubscription struct { - sync.Mutex - name string - lastReadTsNs int64 - cond *sync.Cond -} - -func NewTopicPartitionSubscription(name string) *TopicPartitionSubscription { - t := &TopicPartitionSubscription{ - name: name, - } - t.cond = sync.NewCond(t) - return t -} - -func (s *TopicPartitionSubscription) Wait() { - s.Mutex.Lock() - s.cond.Wait() - s.Mutex.Unlock() -} - -func (s *TopicPartitionSubscription) NotifyOne() { - // notify one waiting goroutine - s.cond.Signal() -} - -type TopicPartitionSubscriptions struct { - sync.Mutex - cond *sync.Cond - subscriptions map[string]*TopicPartitionSubscription - subscriptionsLock sync.RWMutex -} - -func NewTopicPartitionSubscriptions() *TopicPartitionSubscriptions { - m := &TopicPartitionSubscriptions{ - subscriptions: make(map[string]*TopicPartitionSubscription), - } - m.cond = sync.NewCond(m) - return m -} - -func (m *TopicPartitionSubscriptions) AddSubscription(subscription string) *TopicPartitionSubscription { - m.subscriptionsLock.Lock() - defer m.subscriptionsLock.Unlock() - - if s, found := m.subscriptions[subscription]; found { - return s - } - - s := NewTopicPartitionSubscription(subscription) - m.subscriptions[subscription] = s - - return s - -} - -func (m *TopicPartitionSubscriptions) NotifyAll() { - - m.subscriptionsLock.RLock() - defer m.subscriptionsLock.RUnlock() - - for name, tps := range m.subscriptions { - println("notifying", name) - tps.NotifyOne() - } - -} - -func (m *TopicPartitionSubscriptions) Wait() { - m.Mutex.Lock() - m.cond.Wait() - for _, tps := range m.subscriptions { - tps.NotifyOne() - } - m.Mutex.Unlock() -} diff --git a/weed/messaging/broker/topic_manager.go b/weed/messaging/broker/topic_manager.go index e9f8903e8..b563fffa1 100644 --- a/weed/messaging/broker/topic_manager.go +++ b/weed/messaging/broker/topic_manager.go @@ -31,23 +31,22 @@ type TopicControl struct { subscriberCount int publisherCount int logBuffer *log_buffer.LogBuffer - subscriptions *TopicPartitionSubscriptions } type TopicManager struct { sync.Mutex - topicCursors map[TopicPartition]*TopicControl - broker *MessageBroker + topicControls map[TopicPartition]*TopicControl + broker *MessageBroker } func NewTopicManager(messageBroker *MessageBroker) *TopicManager { return &TopicManager{ - topicCursors: make(map[TopicPartition]*TopicControl), - broker: messageBroker, + topicControls: make(map[TopicPartition]*TopicControl), + broker: messageBroker, } } -func (tm *TopicManager) buildLogBuffer(tc *TopicControl, tp TopicPartition, topicConfig *messaging_pb.TopicConfiguration) *log_buffer.LogBuffer { +func (tm *TopicManager) buildLogBuffer(tl *TopicControl, tp TopicPartition, topicConfig *messaging_pb.TopicConfiguration) *log_buffer.LogBuffer { flushFn := func(startTime, stopTime time.Time, buf []byte) { @@ -69,7 +68,7 @@ func (tm *TopicManager) buildLogBuffer(tc *TopicControl, tp TopicPartition, topi } } logBuffer := log_buffer.NewLogBuffer(time.Minute, flushFn, func() { - tc.subscriptions.NotifyAll() + tl.cond.Broadcast() }) return logBuffer @@ -79,11 +78,11 @@ func (tm *TopicManager) RequestLock(partition TopicPartition, topicConfig *messa tm.Lock() defer tm.Unlock() - tc, found := tm.topicCursors[partition] + tc, found := tm.topicControls[partition] if !found { tc = &TopicControl{} - tm.topicCursors[partition] = tc - tc.subscriptions = NewTopicPartitionSubscriptions() + tc.cond = sync.NewCond(&tc.Mutex) + tm.topicControls[partition] = tc tc.logBuffer = tm.buildLogBuffer(tc, partition, topicConfig) } if isPublisher { @@ -98,7 +97,7 @@ func (tm *TopicManager) ReleaseLock(partition TopicPartition, isPublisher bool) tm.Lock() defer tm.Unlock() - lock, found := tm.topicCursors[partition] + lock, found := tm.topicControls[partition] if !found { return } @@ -108,7 +107,7 @@ func (tm *TopicManager) ReleaseLock(partition TopicPartition, isPublisher bool) lock.subscriberCount-- } if lock.subscriberCount <= 0 && lock.publisherCount <= 0 { - delete(tm.topicCursors, partition) + delete(tm.topicControls, partition) lock.logBuffer.Shutdown() } } @@ -117,7 +116,7 @@ func (tm *TopicManager) ListTopicPartitions() (tps []TopicPartition) { tm.Lock() defer tm.Unlock() - for k := range tm.topicCursors { + for k := range tm.topicControls { tps = append(tps, k) } return From 7aafb9e3f800a2cf2ad704b291887b9ea89507d4 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Fri, 15 May 2020 21:39:03 -0700 Subject: [PATCH 14/24] Revert "filer metrics upgrade prometheus client to 1.6.0" This reverts commit 09aef5583922042ac4d566c70708464426751b23. --- go.mod | 9 +++++---- go.sum | 40 ---------------------------------------- 2 files changed, 5 insertions(+), 44 deletions(-) diff --git a/go.mod b/go.mod index b1e443934..2ba8658d3 100644 --- a/go.mod +++ b/go.mod @@ -32,7 +32,7 @@ require ( github.com/gocql/gocql v0.0.0-20190829130954-e163eff7a8c6 github.com/gogo/protobuf v1.2.2-0.20190730201129-28a6bbf47e48 // indirect github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6 // indirect - github.com/golang/protobuf v1.4.0 + github.com/golang/protobuf v1.3.2 github.com/google/btree v1.0.0 github.com/google/uuid v1.1.1 github.com/gorilla/mux v1.7.3 @@ -56,7 +56,8 @@ require ( github.com/onsi/gomega v1.7.0 // indirect github.com/peterh/liner v1.1.0 github.com/pierrec/lz4 v2.2.7+incompatible // indirect - github.com/prometheus/client_golang v1.6.0 + github.com/prometheus/client_golang v1.1.0 + github.com/prometheus/procfs v0.0.4 // indirect github.com/rakyll/statik v0.1.7 github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563 // indirect github.com/seaweedfs/fuse v0.0.0-20190510212405-310228904eff @@ -66,7 +67,7 @@ require ( github.com/spf13/jwalterweatherman v1.1.0 // indirect github.com/spf13/viper v1.4.0 github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271 // indirect - github.com/stretchr/testify v1.4.0 + github.com/stretchr/testify v1.3.0 github.com/syndtr/goleveldb v1.0.0 github.com/tidwall/gjson v1.3.2 github.com/tidwall/match v1.0.1 @@ -83,7 +84,7 @@ require ( golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7 // indirect golang.org/x/image v0.0.0-20200119044424-58c23975cae1 // indirect golang.org/x/net v0.0.0-20190909003024-a7b16738d86b - golang.org/x/sys v0.0.0-20200420163511-1957bb5e6d1f + golang.org/x/sys v0.0.0-20190910064555-bbd175535a8b golang.org/x/tools v0.0.0-20190911022129-16c5e0f7d110 google.golang.org/api v0.9.0 google.golang.org/appengine v1.6.2 // indirect diff --git a/go.sum b/go.sum index 9762df0ea..00329907e 100644 --- a/go.sum +++ b/go.sum @@ -42,9 +42,7 @@ github.com/Shopify/sarama v1.23.1/go.mod h1:XLH1GYJnLVE0XCr6KdJGVJRTwY30moWNJ4sE github.com/Shopify/toxiproxy v2.1.4+incompatible h1:TKdv8HiTLgE5wdJuEML90aBgNWsokNbMijUGhmcoBJc= github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/aws/aws-sdk-go v1.15.27/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZoCYDt7FT0= github.com/aws/aws-sdk-go v1.19.18/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= @@ -68,8 +66,6 @@ github.com/census-instrumentation/opencensus-proto v0.2.1 h1:glEXhBS5PSLLv4IXzLA github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= -github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= -github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chrislusf/raft v0.0.0-20190225081310-10d6e2182d92 h1:lM9SFsh0EPXkyJyrTJqLZPAIJBtNFP6LNkYXu2MnSZI= github.com/chrislusf/raft v0.0.0-20190225081310-10d6e2182d92/go.mod h1:4jyiUCD5y548+yKW+oiHtccBiMaLCCbFBpK2t7X4eUo= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= @@ -134,7 +130,6 @@ github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-ini/ini v1.25.4/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-redis/redis v6.15.7+incompatible h1:3skhDh95XQMpnqeqNftPkQD9jL9e5e36z/1SUm6dy1U= @@ -187,12 +182,6 @@ github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5y github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= -github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= -github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= -github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= -github.com/golang/protobuf v1.4.0 h1:oOuy+ugB+P/kBdUnG5QaMXSIyJ1q38wWSojYCb3z5VQ= -github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= github.com/golang/snappy v0.0.0-20170215233205-553a64147049/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= @@ -205,8 +194,6 @@ github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= -github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-replayers/grpcreplay v0.1.0 h1:eNb1y9rZFmY4ax45uEEECSa8fsxGRU+8Bil52ASAwic= github.com/google/go-replayers/grpcreplay v0.1.0/go.mod h1:8Ig2Idjpr6gifRd6pNVggX6TC1Zw6Jx74AKp7QNH2QE= github.com/google/go-replayers/httpreplay v0.1.0 h1:AX7FUb4BjrrzNvblr/OlgwrmFiep6soj5K2QSDW7BGk= @@ -272,8 +259,6 @@ github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.7 h1:KfgG9LzI+pYjr4xvmz/5H4FXjokeP+rlHLhv3iH62Fo= github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.9 h1:9yzud/Ht36ygwatGx56VwCZtlI/2AD15T1X2sjSuGns= -github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/karlseguin/ccache v2.0.3+incompatible h1:j68C9tWOROiOLWTS/kCGg9IcJG+ACqn5+0+t8Oh83UU= @@ -383,23 +368,17 @@ github.com/prometheus/client_golang v1.0.0 h1:vrDKnkGzuGvhNAL56c7DBz29ZL+KxnoR0x github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.1.0 h1:BQ53HtBmfOitExawJ6LokA4x8ov/z0SYYb0+HxJfRI8= github.com/prometheus/client_golang v1.1.0/go.mod h1:I1FGZT9+L76gKKOs5djB6ezCbFQP1xR9D75/vuwEF3g= -github.com/prometheus/client_golang v1.6.0 h1:YVPodQOcK15POxhgARIvnDRVpLcuK8mglnMrWfyrw6A= -github.com/prometheus/client_golang v1.6.0/go.mod h1:ZLOG9ck3JLRdB5MgO8f+lLTe83AXG6ro35rLTxvnIl4= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90 h1:S/YWwWx/RA8rT8tKFRuGUZhuA90OyIBpPCXkcbwU8DE= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 h1:gQz4mCbXsO+nc9n1hCxHcGA3Zx3Eo+UHZoInFGUIXNM= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= -github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1 h1:K0MGApIoQvMw27RTdJkPbr3JZ7DNbtxQNyi5STVM6Kw= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.6.0 h1:kRhiuYSXR3+uv2IbVbZhUxK5zVD/2pp3Gd2PpvPkpEo= github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc= -github.com/prometheus/common v0.9.1 h1:KOMtN28tlbam3/7ZKEYKHhKoJZYYj3gMH4uc62x7X7U= -github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.2 h1:6LJUbpNm42llc4HRCuvApCSWB/WfhuNo9K98Q9sNGfs= @@ -407,8 +386,6 @@ github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsT github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= github.com/prometheus/procfs v0.0.4 h1:w8DjqFMJDjuVwdZBQoOozr4MVWOnwF7RcL/7uxBjY78= github.com/prometheus/procfs v0.0.4/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= -github.com/prometheus/procfs v0.0.11 h1:DhHlBtkHWPYi8O2y31JkK0TF+DGM+51OopZjH/Ia5qI= -github.com/prometheus/procfs v0.0.11/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/rakyll/statik v0.1.7 h1:OF3QCZUuyPxuGEP7B4ypUa7sB/iHtqOTDYZXGM8KOdQ= github.com/rakyll/statik v0.1.7/go.mod h1:AlZONWzMtEnMs7W4e/1LURLiI49pIMmp6V9Unghqrcc= @@ -462,7 +439,6 @@ github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoH github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE= github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= github.com/tidwall/gjson v1.3.2 h1:+7p3qQFaH3fOMXAJSrdZwGKcOO/lYdGS0HqGhPqDdTI= @@ -569,8 +545,6 @@ golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190412183630-56d357773e84/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY= -golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -594,9 +568,6 @@ golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190910064555-bbd175535a8b h1:3S2h5FadpNr0zUUCVZjlKIEYF+KaX/OBplTGo89CYHI= golang.org/x/sys v0.0.0-20190910064555-bbd175535a8b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200420163511-1957bb5e6d1f h1:gWF768j/LaZugp8dyS4UwsslYCYz9XgFxvlgsn0n9H8= -golang.org/x/sys v0.0.0-20200420163511-1957bb5e6d1f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= @@ -627,8 +598,6 @@ golang.org/x/tools v0.0.0-20190911022129-16c5e0f7d110/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7 h1:9zdDQZ7Thm29KFXgAX/+yaf3eVbP7djjWp/dXAppNCc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.5.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.6.0/go.mod h1:btoxGiFvQNVUZQ8W08zLtrVS08CNpINPEfxXxgJL1Q4= @@ -664,12 +633,6 @@ google.golang.org/grpc v1.23.0 h1:AzbTB6ux+okLTzP8Ru1Xs41C303zdcfEht7MQnYJt5A= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.26.0 h1:2dTRdpdFEEhJYQD8EMLB61nnrzSCTbG38PhqdhvOltg= google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= -google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= -google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= -google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= -google.golang.org/protobuf v1.21.0 h1:qdOKuR/EIArgaWNjetjgTzgVTAZ+S/WXVrq9HW9zimw= -google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -703,9 +666,6 @@ gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bl gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.5 h1:ymVxjfMaHvXD8RqPRmzHHsB3VvucivSkIAvJFDI5O3c= -gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= From b0de01ff3b91cf28edbbaa536eed0e02db4d492c Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sat, 16 May 2020 08:57:29 -0700 Subject: [PATCH 15/24] able to detect disconnected subscribers --- .../broker/broker_grpc_server_subscribe.go | 19 +++++++-- weed/messaging/msgclient/sub_chan.go | 6 ++- weed/messaging/msgclient/subscriber.go | 40 +++++++++---------- 3 files changed, 40 insertions(+), 25 deletions(-) diff --git a/weed/messaging/broker/broker_grpc_server_subscribe.go b/weed/messaging/broker/broker_grpc_server_subscribe.go index e7cbb6441..9e951ce65 100644 --- a/weed/messaging/broker/broker_grpc_server_subscribe.go +++ b/weed/messaging/broker/broker_grpc_server_subscribe.go @@ -50,6 +50,17 @@ func (broker *MessageBroker) Subscribe(stream messaging_pb.SeaweedMessaging_Subs lock := broker.topicManager.RequestLock(tp, topicConfig, false) defer broker.topicManager.ReleaseLock(tp, false) + isConnected := true + go func() { + for isConnected { + time.Sleep(1737 * time.Millisecond) + if err = stream.Send(&messaging_pb.BrokerMessage{}); err != nil { + isConnected = false + lock.cond.Signal() + } + } + }() + lastReadTime := time.Now() switch in.Init.StartPosition { case messaging_pb.SubscriberMessage_InitMessage_TIMESTAMP: @@ -93,8 +104,10 @@ func (broker *MessageBroker) Subscribe(stream messaging_pb.SeaweedMessaging_Subs } if err := broker.readPersistedLogBuffer(&tp, lastReadTime, eachLogEntryFn); err != nil { - // println("stopping from persisted logs") - return err + if err != io.EOF { + println("stopping from persisted logs", err.Error()) + return err + } } if processedTsNs != 0 { @@ -105,7 +118,7 @@ func (broker *MessageBroker) Subscribe(stream messaging_pb.SeaweedMessaging_Subs lock.Mutex.Lock() lock.cond.Wait() lock.Mutex.Unlock() - return true + return isConnected }, eachLogEntryFn) return err diff --git a/weed/messaging/msgclient/sub_chan.go b/weed/messaging/msgclient/sub_chan.go index 5465c5913..3eabc6210 100644 --- a/weed/messaging/msgclient/sub_chan.go +++ b/weed/messaging/msgclient/sub_chan.go @@ -27,7 +27,7 @@ func (mc *MessagingClient) NewSubChannel(subscriberId, chanName string) (*SubCha if err != nil { return nil, err } - sc, err := setupSubscriberClient(grpcConnection, subscriberId, "chan", chanName, 0, time.Unix(0, 0)) + sc, err := setupSubscriberClient(grpcConnection, tp, subscriberId, time.Unix(0, 0)) if err != nil { return nil, err } @@ -48,6 +48,10 @@ func (mc *MessagingClient) NewSubChannel(subscriberId, chanName string) (*SubCha log.Printf("fail to receive from netchan %s: %v", chanName, subErr) return } + if resp.Data == nil { + // this could be heartbeat from broker + continue + } if resp.Data.IsClose { t.stream.Send(&messaging_pb.SubscriberMessage{ IsClose: true, diff --git a/weed/messaging/msgclient/subscriber.go b/weed/messaging/msgclient/subscriber.go index efbfa0337..2e66923e2 100644 --- a/weed/messaging/msgclient/subscriber.go +++ b/weed/messaging/msgclient/subscriber.go @@ -5,6 +5,7 @@ import ( "io" "time" + "github.com/chrislusf/seaweedfs/weed/messaging/broker" "github.com/chrislusf/seaweedfs/weed/pb/messaging_pb" "google.golang.org/grpc" ) @@ -14,7 +15,6 @@ type Subscriber struct { subscriberId string } -/* func (mc *MessagingClient) NewSubscriber(subscriberId, namespace, topic string, startTime time.Time) (*Subscriber, error) { // read topic configuration topicConfiguration := &messaging_pb.TopicConfiguration{ @@ -23,7 +23,16 @@ func (mc *MessagingClient) NewSubscriber(subscriberId, namespace, topic string, subscriberClients := make([]messaging_pb.SeaweedMessaging_SubscribeClient, topicConfiguration.PartitionCount) for i := 0; i < int(topicConfiguration.PartitionCount); i++ { - client, err := mc.setupSubscriberClient(subscriberId, namespace, topic, int32(i), startTime) + tp := broker.TopicPartition{ + Namespace: namespace, + Topic: topic, + Partition: int32(i), + } + grpcClientConn, err := mc.findBroker(tp) + if err != nil { + return nil, err + } + client, err := setupSubscriberClient(grpcClientConn, tp, subscriberId, startTime) if err != nil { return nil, err } @@ -36,22 +45,7 @@ func (mc *MessagingClient) NewSubscriber(subscriberId, namespace, topic string, }, nil } -func (mc *MessagingClient) setupSubscriberClient(subscriberId, namespace, topic string, partition int32, startTime time.Time) (messaging_pb.SeaweedMessaging_SubscribeClient, error) { - - stream, err := setupSubscriberClient(subscriberId, namespace, topic, partition, startTime) - if err != nil { - return stream, err - } - if newBroker != nil { - - } - - return stream, nil - -} -*/ - -func setupSubscriberClient(grpcConnection *grpc.ClientConn, subscriberId string, namespace string, topic string, partition int32, startTime time.Time) (stream messaging_pb.SeaweedMessaging_SubscribeClient, err error) { +func setupSubscriberClient(grpcConnection *grpc.ClientConn, tp broker.TopicPartition, subscriberId string, startTime time.Time) (stream messaging_pb.SeaweedMessaging_SubscribeClient, err error) { stream, err = messaging_pb.NewSeaweedMessagingClient(grpcConnection).Subscribe(context.Background()) if err != nil { return @@ -60,9 +54,9 @@ func setupSubscriberClient(grpcConnection *grpc.ClientConn, subscriberId string, // send init message err = stream.Send(&messaging_pb.SubscriberMessage{ Init: &messaging_pb.SubscriberMessage_InitMessage{ - Namespace: namespace, - Topic: topic, - Partition: partition, + Namespace: tp.Namespace, + Topic: tp.Topic, + Partition: tp.Partition, StartPosition: messaging_pb.SubscriberMessage_InitMessage_TIMESTAMP, TimestampNs: startTime.UnixNano(), SubscriberId: subscriberId, @@ -90,6 +84,10 @@ func (s *Subscriber) doSubscribe(partition int, processFn func(m *messaging_pb.M println(listenErr.Error()) return listenErr } + if resp.Data == nil { + // this could be heartbeat from broker + continue + } processFn(resp.Data) } } From 759cda0fe297770847b3a2db640bbcaba324f38d Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sat, 16 May 2020 18:47:41 -0700 Subject: [PATCH 16/24] log --- weed/util/log_buffer/log_read.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/weed/util/log_buffer/log_read.go b/weed/util/log_buffer/log_read.go index a2345acfb..2b73a8064 100644 --- a/weed/util/log_buffer/log_read.go +++ b/weed/util/log_buffer/log_read.go @@ -30,6 +30,7 @@ func (logBuffer *LogBuffer) LoopProcessLogData( logBuffer.ReleaseMeory(bytesBuf) } bytesBuf = logBuffer.ReadFromBuffer(lastReadTime) + // fmt.Printf("ReadFromBuffer by %v\n", lastReadTime) if bytesBuf == nil { if waitForDataFn() { continue @@ -39,6 +40,7 @@ func (logBuffer *LogBuffer) LoopProcessLogData( } buf := bytesBuf.Bytes() + // fmt.Printf("ReadFromBuffer by %v size %d\n", lastReadTime, len(buf)) batchSize := 0 var startReadTime time.Time From 85b53ac510aca494f8a3d18bb15b829971795b15 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sat, 16 May 2020 18:53:54 -0700 Subject: [PATCH 17/24] detect disconnected subscribers --- .../broker/broker_grpc_server_subscribe.go | 34 +++++++++++++------ weed/messaging/msgclient/subscriber.go | 18 +++++----- 2 files changed, 32 insertions(+), 20 deletions(-) diff --git a/weed/messaging/broker/broker_grpc_server_subscribe.go b/weed/messaging/broker/broker_grpc_server_subscribe.go index 9e951ce65..d8e8faa31 100644 --- a/weed/messaging/broker/broker_grpc_server_subscribe.go +++ b/weed/messaging/broker/broker_grpc_server_subscribe.go @@ -4,6 +4,7 @@ import ( "fmt" "io" "strings" + "sync" "time" "github.com/golang/protobuf/proto" @@ -25,39 +26,47 @@ func (broker *MessageBroker) Subscribe(stream messaging_pb.SeaweedMessaging_Subs return err } + var processedTsNs int64 var messageCount int64 subscriberId := in.Init.SubscriberId - fmt.Printf("+ subscriber %s\n", subscriberId) - defer func() { - fmt.Printf("- subscriber %s: %d messages\n", subscriberId, messageCount) - }() // TODO look it up topicConfig := &messaging_pb.TopicConfiguration{ // IsTransient: true, } - if err = stream.Send(&messaging_pb.BrokerMessage{}); err != nil { - return err - } - // get lock tp := TopicPartition{ Namespace: in.Init.Namespace, Topic: in.Init.Topic, Partition: in.Init.Partition, } + fmt.Printf("+ subscriber %s for %s\n", subscriberId, tp.String()) + defer func() { + fmt.Printf("- subscriber %s for %s %d messages last %v\n", subscriberId, tp.String(), messageCount, time.Unix(0, processedTsNs)) + }() + lock := broker.topicManager.RequestLock(tp, topicConfig, false) defer broker.topicManager.ReleaseLock(tp, false) isConnected := true + var streamLock sync.Mutex // https://github.com/grpc/grpc-go/issues/948 go func() { + lastActiveTime := time.Now().UnixNano() + sleepTime := 1737 * time.Millisecond for isConnected { - time.Sleep(1737 * time.Millisecond) + time.Sleep(sleepTime) + if lastActiveTime != processedTsNs { + lastActiveTime = processedTsNs + continue + } + streamLock.Lock() + // println("checking connection health to", subscriberId, tp.String()) if err = stream.Send(&messaging_pb.BrokerMessage{}); err != nil { isConnected = false lock.cond.Signal() } + streamLock.Unlock() } }() @@ -69,14 +78,15 @@ func (broker *MessageBroker) Subscribe(stream messaging_pb.SeaweedMessaging_Subs case messaging_pb.SubscriberMessage_InitMessage_EARLIEST: lastReadTime = time.Unix(0, 0) } - var processedTsNs int64 // how to process each message // an error returned will end the subscription eachMessageFn := func(m *messaging_pb.Message) error { + streamLock.Lock() err := stream.Send(&messaging_pb.BrokerMessage{ Data: m, }) + streamLock.Unlock() if err != nil { glog.V(0).Infof("=> subscriber %v: %+v", subscriberId, err) } @@ -105,7 +115,7 @@ func (broker *MessageBroker) Subscribe(stream messaging_pb.SeaweedMessaging_Subs if err := broker.readPersistedLogBuffer(&tp, lastReadTime, eachLogEntryFn); err != nil { if err != io.EOF { - println("stopping from persisted logs", err.Error()) + // println("stopping from persisted logs", err.Error()) return err } } @@ -114,6 +124,8 @@ func (broker *MessageBroker) Subscribe(stream messaging_pb.SeaweedMessaging_Subs lastReadTime = time.Unix(0, processedTsNs) } + // fmt.Printf("subscriber %s read %d on disk log %v\n", subscriberId, messageCount, lastReadTime) + err = lock.logBuffer.LoopProcessLogData(lastReadTime, func() bool { lock.Mutex.Lock() lock.cond.Wait() diff --git a/weed/messaging/msgclient/subscriber.go b/weed/messaging/msgclient/subscriber.go index 2e66923e2..f96bba2ec 100644 --- a/weed/messaging/msgclient/subscriber.go +++ b/weed/messaging/msgclient/subscriber.go @@ -15,7 +15,7 @@ type Subscriber struct { subscriberId string } -func (mc *MessagingClient) NewSubscriber(subscriberId, namespace, topic string, startTime time.Time) (*Subscriber, error) { +func (mc *MessagingClient) NewSubscriber(subscriberId, namespace, topic string, partitionId int, startTime time.Time) (*Subscriber, error) { // read topic configuration topicConfiguration := &messaging_pb.TopicConfiguration{ PartitionCount: 4, @@ -23,6 +23,9 @@ func (mc *MessagingClient) NewSubscriber(subscriberId, namespace, topic string, subscriberClients := make([]messaging_pb.SeaweedMessaging_SubscribeClient, topicConfiguration.PartitionCount) for i := 0; i < int(topicConfiguration.PartitionCount); i++ { + if partitionId>=0 && i != partitionId { + continue + } tp := broker.TopicPartition{ Namespace: namespace, Topic: topic, @@ -66,17 +69,12 @@ func setupSubscriberClient(grpcConnection *grpc.ClientConn, tp broker.TopicParti return } - // process init response - _, err = stream.Recv() - if err != nil { - return - } return stream, nil } -func (s *Subscriber) doSubscribe(partition int, processFn func(m *messaging_pb.Message)) error { +func doSubscribe(subscriberClient messaging_pb.SeaweedMessaging_SubscribeClient, processFn func(m *messaging_pb.Message)) error { for { - resp, listenErr := s.subscriberClients[partition].Recv() + resp, listenErr := subscriberClient.Recv() if listenErr == io.EOF { return nil } @@ -95,6 +93,8 @@ func (s *Subscriber) doSubscribe(partition int, processFn func(m *messaging_pb.M // Subscribe starts goroutines to process the messages func (s *Subscriber) Subscribe(processFn func(m *messaging_pb.Message)) { for i := 0; i < len(s.subscriberClients); i++ { - go s.doSubscribe(i, processFn) + if s.subscriberClients[i] != nil { + go doSubscribe(s.subscriberClients[i], processFn) + } } } From 1ea9bc66d96030720e465bd283ad1eeff16724da Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sun, 17 May 2020 08:56:33 -0700 Subject: [PATCH 18/24] avoid leaking grpc.NewClientStream --- weed/pb/filer_pb/filer_client.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/weed/pb/filer_pb/filer_client.go b/weed/pb/filer_pb/filer_client.go index 73b66472d..d42e20b34 100644 --- a/weed/pb/filer_pb/filer_client.go +++ b/weed/pb/filer_pb/filer_client.go @@ -83,10 +83,12 @@ func doList(filerClient FilerClient, fullDirPath util.FullPath, prefix string, f } glog.V(3).Infof("read directory: %v", request) - stream, err := client.ListEntries(context.Background(), request) + ctx, cancel := context.WithCancel(context.Background()) + stream, err := client.ListEntries(ctx, request) if err != nil { return fmt.Errorf("list %s: %v", fullDirPath, err) } + defer cancel() var prevEntry *Entry for { From f11233cd494b3092753b302166badbefe6bf401a Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sun, 17 May 2020 08:57:47 -0700 Subject: [PATCH 19/24] simplify disconnected stream detection --- .../broker/broker_grpc_server_subscribe.go | 17 ++--------------- 1 file changed, 2 insertions(+), 15 deletions(-) diff --git a/weed/messaging/broker/broker_grpc_server_subscribe.go b/weed/messaging/broker/broker_grpc_server_subscribe.go index d8e8faa31..1065309d2 100644 --- a/weed/messaging/broker/broker_grpc_server_subscribe.go +++ b/weed/messaging/broker/broker_grpc_server_subscribe.go @@ -4,7 +4,6 @@ import ( "fmt" "io" "strings" - "sync" "time" "github.com/golang/protobuf/proto" @@ -50,23 +49,13 @@ func (broker *MessageBroker) Subscribe(stream messaging_pb.SeaweedMessaging_Subs defer broker.topicManager.ReleaseLock(tp, false) isConnected := true - var streamLock sync.Mutex // https://github.com/grpc/grpc-go/issues/948 go func() { - lastActiveTime := time.Now().UnixNano() - sleepTime := 1737 * time.Millisecond for isConnected { - time.Sleep(sleepTime) - if lastActiveTime != processedTsNs { - lastActiveTime = processedTsNs - continue - } - streamLock.Lock() - // println("checking connection health to", subscriberId, tp.String()) - if err = stream.Send(&messaging_pb.BrokerMessage{}); err != nil { + if _, err := stream.Recv(); err != nil { + println("disconnecting connection to", subscriberId, tp.String()) isConnected = false lock.cond.Signal() } - streamLock.Unlock() } }() @@ -82,11 +71,9 @@ func (broker *MessageBroker) Subscribe(stream messaging_pb.SeaweedMessaging_Subs // how to process each message // an error returned will end the subscription eachMessageFn := func(m *messaging_pb.Message) error { - streamLock.Lock() err := stream.Send(&messaging_pb.BrokerMessage{ Data: m, }) - streamLock.Unlock() if err != nil { glog.V(0).Infof("=> subscriber %v: %+v", subscriberId, err) } From 95ca9dd8a2de6af3eb030880123dded9ed6de602 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sun, 17 May 2020 11:10:45 -0700 Subject: [PATCH 20/24] subscribe support cancel --- weed/messaging/msgclient/sub_chan.go | 11 ++++++++++- weed/messaging/msgclient/subscriber.go | 19 ++++++++++++++++--- 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/weed/messaging/msgclient/sub_chan.go b/weed/messaging/msgclient/sub_chan.go index 3eabc6210..213ff4666 100644 --- a/weed/messaging/msgclient/sub_chan.go +++ b/weed/messaging/msgclient/sub_chan.go @@ -1,6 +1,7 @@ package msgclient import ( + "context" "crypto/md5" "hash" "io" @@ -15,6 +16,7 @@ type SubChannel struct { ch chan []byte stream messaging_pb.SeaweedMessaging_SubscribeClient md5hash hash.Hash + cancel context.CancelFunc } func (mc *MessagingClient) NewSubChannel(subscriberId, chanName string) (*SubChannel, error) { @@ -27,7 +29,8 @@ func (mc *MessagingClient) NewSubChannel(subscriberId, chanName string) (*SubCha if err != nil { return nil, err } - sc, err := setupSubscriberClient(grpcConnection, tp, subscriberId, time.Unix(0, 0)) + ctx, cancel := context.WithCancel(context.Background()) + sc, err := setupSubscriberClient(ctx, grpcConnection, tp, subscriberId, time.Unix(0, 0)) if err != nil { return nil, err } @@ -36,6 +39,7 @@ func (mc *MessagingClient) NewSubChannel(subscriberId, chanName string) (*SubCha ch: make(chan []byte), stream: sc, md5hash: md5.New(), + cancel: cancel, } go func() { @@ -57,6 +61,7 @@ func (mc *MessagingClient) NewSubChannel(subscriberId, chanName string) (*SubCha IsClose: true, }) close(t.ch) + cancel() return } t.ch <- resp.Data.Value @@ -74,3 +79,7 @@ func (sc *SubChannel) Channel() chan []byte { func (sc *SubChannel) Md5() []byte { return sc.md5hash.Sum(nil) } + +func (sc *SubChannel) Cancel() { + sc.cancel() +} diff --git a/weed/messaging/msgclient/subscriber.go b/weed/messaging/msgclient/subscriber.go index f96bba2ec..926e193dd 100644 --- a/weed/messaging/msgclient/subscriber.go +++ b/weed/messaging/msgclient/subscriber.go @@ -12,6 +12,7 @@ import ( type Subscriber struct { subscriberClients []messaging_pb.SeaweedMessaging_SubscribeClient + subscriberCancels []context.CancelFunc subscriberId string } @@ -21,6 +22,7 @@ func (mc *MessagingClient) NewSubscriber(subscriberId, namespace, topic string, PartitionCount: 4, } subscriberClients := make([]messaging_pb.SeaweedMessaging_SubscribeClient, topicConfiguration.PartitionCount) + subscriberCancels := make([]context.CancelFunc, topicConfiguration.PartitionCount) for i := 0; i < int(topicConfiguration.PartitionCount); i++ { if partitionId>=0 && i != partitionId { @@ -35,21 +37,24 @@ func (mc *MessagingClient) NewSubscriber(subscriberId, namespace, topic string, if err != nil { return nil, err } - client, err := setupSubscriberClient(grpcClientConn, tp, subscriberId, startTime) + ctx, cancel := context.WithCancel(context.Background()) + client, err := setupSubscriberClient(ctx, grpcClientConn, tp, subscriberId, startTime) if err != nil { return nil, err } subscriberClients[i] = client + subscriberCancels[i] = cancel } return &Subscriber{ subscriberClients: subscriberClients, + subscriberCancels: subscriberCancels, subscriberId: subscriberId, }, nil } -func setupSubscriberClient(grpcConnection *grpc.ClientConn, tp broker.TopicPartition, subscriberId string, startTime time.Time) (stream messaging_pb.SeaweedMessaging_SubscribeClient, err error) { - stream, err = messaging_pb.NewSeaweedMessagingClient(grpcConnection).Subscribe(context.Background()) +func setupSubscriberClient(ctx context.Context, grpcConnection *grpc.ClientConn, tp broker.TopicPartition, subscriberId string, startTime time.Time) (stream messaging_pb.SeaweedMessaging_SubscribeClient, err error) { + stream, err = messaging_pb.NewSeaweedMessagingClient(grpcConnection).Subscribe(ctx) if err != nil { return } @@ -98,3 +103,11 @@ func (s *Subscriber) Subscribe(processFn func(m *messaging_pb.Message)) { } } } + +func (s *Subscriber) Shutdown() { + for i := 0; i < len(s.subscriberClients); i++ { + if s.subscriberCancels[i] != nil { + s.subscriberCancels[i]() + } + } +} From 3a57aef7a910ba66d6026f73d1b3055c2d8a1e4d Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sun, 17 May 2020 17:33:53 -0700 Subject: [PATCH 21/24] sync subscribe() --- weed/messaging/msgclient/subscriber.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/weed/messaging/msgclient/subscriber.go b/weed/messaging/msgclient/subscriber.go index 926e193dd..01e63df40 100644 --- a/weed/messaging/msgclient/subscriber.go +++ b/weed/messaging/msgclient/subscriber.go @@ -4,6 +4,7 @@ import ( "context" "io" "time" + "sync" "github.com/chrislusf/seaweedfs/weed/messaging/broker" "github.com/chrislusf/seaweedfs/weed/pb/messaging_pb" @@ -97,11 +98,17 @@ func doSubscribe(subscriberClient messaging_pb.SeaweedMessaging_SubscribeClient, // Subscribe starts goroutines to process the messages func (s *Subscriber) Subscribe(processFn func(m *messaging_pb.Message)) { + var wg sync.WaitGroup for i := 0; i < len(s.subscriberClients); i++ { if s.subscriberClients[i] != nil { - go doSubscribe(s.subscriberClients[i], processFn) + wg.Add(1) + go func() { + defer wg.Done() + doSubscribe(s.subscriberClients[i], processFn) + }() } } + wg.Wait() } func (s *Subscriber) Shutdown() { From f5684839a2799656c0d8ddf68e191b2c4e29ed30 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sun, 17 May 2020 17:34:10 -0700 Subject: [PATCH 22/24] add DeleteTopic --- weed/messaging/msgclient/config.go | 63 ++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) create mode 100644 weed/messaging/msgclient/config.go diff --git a/weed/messaging/msgclient/config.go b/weed/messaging/msgclient/config.go new file mode 100644 index 000000000..2b9eba1a8 --- /dev/null +++ b/weed/messaging/msgclient/config.go @@ -0,0 +1,63 @@ +package msgclient + +import ( + "context" + "log" + + "github.com/chrislusf/seaweedfs/weed/messaging/broker" + "github.com/chrislusf/seaweedfs/weed/pb" + "github.com/chrislusf/seaweedfs/weed/pb/messaging_pb" +) + +func (mc *MessagingClient) configureTopic(tp broker.TopicPartition) error { + + return mc.withAnyBroker(func(client messaging_pb.SeaweedMessagingClient) error { + _, err := client.ConfigureTopic(context.Background(), + &messaging_pb.ConfigureTopicRequest{ + Namespace: tp.Namespace, + Topic: tp.Topic, + Configuration: &messaging_pb.TopicConfiguration{ + PartitionCount: 0, + Collection: "", + Replication: "", + IsTransient: false, + Partitoning: 0, + }, + }) + return err + }) + +} + +func (mc *MessagingClient) DeleteTopic(namespace, topic string) error { + + return mc.withAnyBroker(func(client messaging_pb.SeaweedMessagingClient) error { + _, err := client.DeleteTopic(context.Background(), + &messaging_pb.DeleteTopicRequest{ + Namespace: namespace, + Topic: topic, + }) + return err + }) +} + +func (mc *MessagingClient) withAnyBroker(fn func(client messaging_pb.SeaweedMessagingClient) error) error { + + var lastErr error + for _, broker := range mc.bootstrapBrokers { + grpcConnection, err := pb.GrpcDial(context.Background(), broker, mc.grpcDialOption) + if err != nil { + log.Printf("dial broker %s: %v", broker, err) + continue + } + defer grpcConnection.Close() + + err = fn(messaging_pb.NewSeaweedMessagingClient(grpcConnection)) + if err == nil { + return nil + } + lastErr = err + } + + return lastErr +} From bff09fb74fbcd49ce13b282ed92919769da56377 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sun, 17 May 2020 17:38:21 -0700 Subject: [PATCH 23/24] fix looping --- weed/messaging/msgclient/subscriber.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/weed/messaging/msgclient/subscriber.go b/weed/messaging/msgclient/subscriber.go index 01e63df40..caa795626 100644 --- a/weed/messaging/msgclient/subscriber.go +++ b/weed/messaging/msgclient/subscriber.go @@ -102,10 +102,10 @@ func (s *Subscriber) Subscribe(processFn func(m *messaging_pb.Message)) { for i := 0; i < len(s.subscriberClients); i++ { if s.subscriberClients[i] != nil { wg.Add(1) - go func() { + go func(subscriberClient messaging_pb.SeaweedMessaging_SubscribeClient) { defer wg.Done() - doSubscribe(s.subscriberClients[i], processFn) - }() + doSubscribe(subscriberClient, processFn) + }(s.subscriberClients[i]) } } wg.Wait() From 081ee6fe349b519da8ea54cf3cdc17d2b15c5a71 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sun, 17 May 2020 17:38:31 -0700 Subject: [PATCH 24/24] reduc logs --- weed/messaging/broker/broker_grpc_server_subscribe.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/weed/messaging/broker/broker_grpc_server_subscribe.go b/weed/messaging/broker/broker_grpc_server_subscribe.go index 1065309d2..9538d3063 100644 --- a/weed/messaging/broker/broker_grpc_server_subscribe.go +++ b/weed/messaging/broker/broker_grpc_server_subscribe.go @@ -52,7 +52,7 @@ func (broker *MessageBroker) Subscribe(stream messaging_pb.SeaweedMessaging_Subs go func() { for isConnected { if _, err := stream.Recv(); err != nil { - println("disconnecting connection to", subscriberId, tp.String()) + // println("disconnecting connection to", subscriberId, tp.String()) isConnected = false lock.cond.Signal() }