Mirror
/
seaweedfs
mirror of https://github.com/seaweedfs/seaweedfs.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

subscriber find broker leader first

pull/5637/head
Chris Lu 11 months ago
parent
e27deed4bc
commit
16b4f9d9ab
4 changed files with 43 additions and 38 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 16
      weed/mq/broker/broker_grpc_admin.go
  2. 6
      weed/mq/client/cmd/weed_sub/subscriber.go
  3. 49
      weed/mq/client/sub_client/connect_to_sub_coordinator.go
  4. 10
      weed/mq/client/sub_client/subscriber.go

16
weed/mq/broker/broker_grpc_admin.go
View File

@ -2,25 +2,21 @@ package broker
import ( import (
"context" "context"
"github.com/seaweedfs/seaweedfs/weed/cluster"
"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
"github.com/seaweedfs/seaweedfs/weed/mq/pub_balancer"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"github.com/seaweedfs/seaweedfs/weed/pb/mq_pb" "github.com/seaweedfs/seaweedfs/weed/pb/mq_pb"
) )
func (b *MessageQueueBroker) FindBrokerLeader(c context.Context, request *mq_pb.FindBrokerLeaderRequest) (*mq_pb.FindBrokerLeaderResponse, error) { func (b *MessageQueueBroker) FindBrokerLeader(c context.Context, request *mq_pb.FindBrokerLeaderRequest) (*mq_pb.FindBrokerLeaderResponse, error) {
ret := &mq_pb.FindBrokerLeaderResponse{} ret := &mq_pb.FindBrokerLeaderResponse{}
err := b.withMasterClient(false, b.MasterClient.GetMaster(), func(client master_pb.SeaweedClient) error {
resp, err := client.ListClusterNodes(context.Background(), &master_pb.ListClusterNodesRequest{
ClientType: cluster.BrokerType,
FilerGroup: request.FilerGroup,
err := b.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {
resp, err := client.FindLockOwner(context.Background(), &filer_pb.FindLockOwnerRequest{
Name: pub_balancer.LockBrokerBalancer,
}) })
if err != nil { if err != nil {
return err return err
} }
if len(resp.ClusterNodes) == 0 {
return nil
}
ret.Broker = resp.ClusterNodes[0].Address
ret.Broker = resp.Owner
return nil return nil
}) })
return ret, err return ret, err

6
weed/mq/client/cmd/weed_sub/subscriber.go
View File

@ -5,6 +5,7 @@ import (
"fmt" "fmt"
"github.com/seaweedfs/seaweedfs/weed/glog" "github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/mq/client/sub_client" "github.com/seaweedfs/seaweedfs/weed/mq/client/sub_client"
"github.com/seaweedfs/seaweedfs/weed/mq/topic"
"github.com/seaweedfs/seaweedfs/weed/util" "github.com/seaweedfs/seaweedfs/weed/util"
"google.golang.org/grpc" "google.golang.org/grpc"
"google.golang.org/grpc/credentials/insecure" "google.golang.org/grpc/credentials/insecure"
@ -14,7 +15,7 @@ import (
var ( var (
namespace = flag.String("ns", "test", "namespace") namespace = flag.String("ns", "test", "namespace")
topic = flag.String("topic", "test", "topic")
t = flag.String("topic", "test", "topic")
seedBrokers = flag.String("brokers", "localhost:17777", "seed brokers") seedBrokers = flag.String("brokers", "localhost:17777", "seed brokers")
clientId = flag.Uint("client_id", uint(util.RandomInt32()), "client id") clientId = flag.Uint("client_id", uint(util.RandomInt32()), "client id")
@ -31,8 +32,7 @@ func main() {
} }
contentConfig := &sub_client.ContentConfiguration{ contentConfig := &sub_client.ContentConfiguration{
Namespace: *namespace,
Topic: *topic,
Topic: topic.NewTopic(*namespace, *t),
Filter: "", Filter: "",
StartTime: time.Unix(1, 1), StartTime: time.Unix(1, 1),
} }

49
weed/mq/client/sub_client/connect_to_sub_coordinator.go
View File

@ -15,15 +15,30 @@ func (sub *TopicSubscriber) doKeepConnectedToSubCoordinator() {
waitTime := 1 * time.Second waitTime := 1 * time.Second
for { for {
for _, broker := range sub.bootstrapBrokers { for _, broker := range sub.bootstrapBrokers {
// TODO find the balancer
// lookup topic brokers
var brokerLeader string
err := pb.WithBrokerGrpcClient(false, broker, sub.SubscriberConfig.GrpcDialOption, func(client mq_pb.SeaweedMessagingClient) error {
resp, err := client.FindBrokerLeader(context.Background(), &mq_pb.FindBrokerLeaderRequest{})
if err != nil {
return err
}
brokerLeader = resp.Broker
return nil
})
if err != nil {
glog.V(0).Infof("broker coordinator on %s: %v", broker, err)
continue
}
glog.V(0).Infof("found broker coordinator: %v", brokerLeader)
// connect to the balancer // connect to the balancer
pb.WithBrokerGrpcClient(true, broker, sub.SubscriberConfig.GrpcDialOption, func(client mq_pb.SeaweedMessagingClient) error {
pb.WithBrokerGrpcClient(true, brokerLeader, sub.SubscriberConfig.GrpcDialOption, func(client mq_pb.SeaweedMessagingClient) error {
ctx, cancel := context.WithCancel(context.Background()) ctx, cancel := context.WithCancel(context.Background())
defer cancel() defer cancel()
stream, err := client.SubscriberToSubCoordinator(ctx) stream, err := client.SubscriberToSubCoordinator(ctx)
if err != nil { if err != nil {
glog.V(0).Infof("subscriber %s/%s: %v", sub.ContentConfig.Namespace, sub.ContentConfig.Topic, err)
glog.V(0).Infof("subscriber %s: %v", sub.ContentConfig.Topic, err)
return err return err
} }
waitTime = 1 * time.Second waitTime = 1 * time.Second
@ -33,16 +48,13 @@ func (sub *TopicSubscriber) doKeepConnectedToSubCoordinator() {
if err := stream.Send(&mq_pb.SubscriberToSubCoordinatorRequest{ if err := stream.Send(&mq_pb.SubscriberToSubCoordinatorRequest{
Message: &mq_pb.SubscriberToSubCoordinatorRequest_Init{ Message: &mq_pb.SubscriberToSubCoordinatorRequest_Init{
Init: &mq_pb.SubscriberToSubCoordinatorRequest_InitMessage{ Init: &mq_pb.SubscriberToSubCoordinatorRequest_InitMessage{
ConsumerGroup: sub.SubscriberConfig.ConsumerGroup,
ConsumerGroup: sub.SubscriberConfig.ConsumerGroup,
ConsumerGroupInstanceId: sub.SubscriberConfig.ConsumerGroupInstanceId, ConsumerGroupInstanceId: sub.SubscriberConfig.ConsumerGroupInstanceId,
Topic: &mq_pb.Topic{
Namespace: sub.ContentConfig.Namespace,
Name: sub.ContentConfig.Topic,
},
Topic: sub.ContentConfig.Topic.ToPbTopic(),
}, },
}, },
}); err != nil { }); err != nil {
glog.V(0).Infof("subscriber %s/%s send init: %v", sub.ContentConfig.Namespace, sub.ContentConfig.Topic, err)
glog.V(0).Infof("subscriber %s send init: %v", sub.ContentConfig.Topic, err)
return err return err
} }
@ -50,12 +62,12 @@ func (sub *TopicSubscriber) doKeepConnectedToSubCoordinator() {
for { for {
resp, err := stream.Recv() resp, err := stream.Recv()
if err != nil { if err != nil {
glog.V(0).Infof("subscriber %s/%s receive: %v", sub.ContentConfig.Namespace, sub.ContentConfig.Topic, err)
glog.V(0).Infof("subscriber %s receive: %v", sub.ContentConfig.Topic, err)
return err return err
} }
assignment := resp.GetAssignment() assignment := resp.GetAssignment()
if assignment != nil { if assignment != nil {
glog.V(0).Infof("subscriber %s/%s receive assignment: %v", sub.ContentConfig.Namespace, sub.ContentConfig.Topic, assignment)
glog.V(0).Infof("subscriber %s receive assignment: %v", sub.ContentConfig.Topic, assignment)
} }
sub.onEachAssignment(assignment) sub.onEachAssignment(assignment)
} }
@ -63,7 +75,7 @@ func (sub *TopicSubscriber) doKeepConnectedToSubCoordinator() {
return nil return nil
}) })
} }
glog.V(0).Infof("subscriber %s/%s/%s waiting for more assignments", sub.ContentConfig.Namespace, sub.ContentConfig.Topic, sub.SubscriberConfig.ConsumerGroup)
glog.V(0).Infof("subscriber %s/%s waiting for more assignments", sub.ContentConfig.Topic, sub.SubscriberConfig.ConsumerGroup)
if waitTime < 10*time.Second { if waitTime < 10*time.Second {
waitTime += 1 * time.Second waitTime += 1 * time.Second
} }
@ -85,10 +97,10 @@ func (sub *TopicSubscriber) onEachAssignment(assignment *mq_pb.SubscriberToSubCo
go func(partition *mq_pb.Partition, broker string) { go func(partition *mq_pb.Partition, broker string) {
defer wg.Done() defer wg.Done()
defer func() { <-semaphore }() defer func() { <-semaphore }()
glog.V(0).Infof("subscriber %s/%s/%s assigned partition %+v at %v", sub.ContentConfig.Namespace, sub.ContentConfig.Topic, sub.SubscriberConfig.ConsumerGroup, partition, broker)
glog.V(0).Infof("subscriber %s/%s assigned partition %+v at %v", sub.ContentConfig.Topic, sub.SubscriberConfig.ConsumerGroup, partition, broker)
err := sub.onEachPartition(partition, broker) err := sub.onEachPartition(partition, broker)
if err != nil { if err != nil {
glog.V(0).Infof("subscriber %s/%s/%s partition %+v at %v: %v", sub.ContentConfig.Namespace, sub.ContentConfig.Topic, sub.SubscriberConfig.ConsumerGroup, partition, broker, err)
glog.V(0).Infof("subscriber %s/%s partition %+v at %v: %v", sub.ContentConfig.Topic, sub.SubscriberConfig.ConsumerGroup, partition, broker, err)
} }
}(assigned.Partition, assigned.Broker) }(assigned.Partition, assigned.Broker)
} }
@ -104,10 +116,7 @@ func (sub *TopicSubscriber) onEachPartition(partition *mq_pb.Partition, broker s
Init: &mq_pb.SubscribeMessageRequest_InitMessage{ Init: &mq_pb.SubscribeMessageRequest_InitMessage{
ConsumerGroup: sub.SubscriberConfig.ConsumerGroup, ConsumerGroup: sub.SubscriberConfig.ConsumerGroup,
ConsumerId: sub.SubscriberConfig.ConsumerGroupInstanceId, ConsumerId: sub.SubscriberConfig.ConsumerGroupInstanceId,
Topic: &mq_pb.Topic{
Namespace: sub.ContentConfig.Namespace,
Name: sub.ContentConfig.Topic,
},
Topic: sub.ContentConfig.Topic.ToPbTopic(),
PartitionOffset: &mq_pb.PartitionOffset{ PartitionOffset: &mq_pb.PartitionOffset{
Partition: partition, Partition: partition,
StartTsNs: sub.alreadyProcessedTsNs, StartTsNs: sub.alreadyProcessedTsNs,
@ -122,7 +131,7 @@ func (sub *TopicSubscriber) onEachPartition(partition *mq_pb.Partition, broker s
return fmt.Errorf("create subscribe client: %v", err) return fmt.Errorf("create subscribe client: %v", err)
} }
glog.V(0).Infof("subscriber %s/%s/%s connected to partition %+v at %v", sub.ContentConfig.Namespace, sub.ContentConfig.Topic, sub.SubscriberConfig.ConsumerGroup, partition, broker)
glog.V(0).Infof("subscriber %s/%s connected to partition %+v at %v", sub.ContentConfig.Topic, sub.SubscriberConfig.ConsumerGroup, partition, broker)
if sub.OnCompletionFunc != nil { if sub.OnCompletionFunc != nil {
defer sub.OnCompletionFunc() defer sub.OnCompletionFunc()
@ -145,7 +154,7 @@ func (sub *TopicSubscriber) onEachPartition(partition *mq_pb.Partition, broker s
return fmt.Errorf("subscribe recv: %v", err) return fmt.Errorf("subscribe recv: %v", err)
} }
if resp.Message == nil { if resp.Message == nil {
glog.V(0).Infof("subscriber %s/%s/%s received nil message", sub.ContentConfig.Namespace, sub.ContentConfig.Topic, sub.SubscriberConfig.ConsumerGroup)
glog.V(0).Infof("subscriber %s/%s received nil message", sub.ContentConfig.Topic, sub.SubscriberConfig.ConsumerGroup)
continue continue
} }
switch m := resp.Message.(type) { switch m := resp.Message.(type) {

10
weed/mq/client/sub_client/subscriber.go
View File

@ -1,6 +1,7 @@
package sub_client package sub_client
import ( import (
"github.com/seaweedfs/seaweedfs/weed/mq/topic"
"github.com/seaweedfs/seaweedfs/weed/pb/mq_pb" "github.com/seaweedfs/seaweedfs/weed/pb/mq_pb"
"google.golang.org/grpc" "google.golang.org/grpc"
"time" "time"
@ -11,14 +12,13 @@ type SubscriberConfiguration struct {
ConsumerGroup string ConsumerGroup string
ConsumerGroupInstanceId string ConsumerGroupInstanceId string
GroupMinimumPeers int32 GroupMinimumPeers int32
GroupMaximumPeers int32
BootstrapServers []string
GrpcDialOption grpc.DialOption
GroupMaximumPeers int32
BootstrapServers []string
GrpcDialOption grpc.DialOption
} }
type ContentConfiguration struct { type ContentConfiguration struct {
Namespace string
Topic string
Topic topic.Topic
Filter string Filter string
StartTime time.Time StartTime time.Time
} }

Write Preview
Loading…
Cancel
Save