You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

83 lines
1.9 KiB

6 years ago
6 years ago
6 years ago
  1. package replication
  2. import (
  3. "fmt"
  4. "github.com/Shopify/sarama"
  5. "github.com/chrislusf/seaweedfs/weed/glog"
  6. "github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
  7. "github.com/chrislusf/seaweedfs/weed/util"
  8. "github.com/golang/protobuf/proto"
  9. )
  10. func init() {
  11. NotificationInputs = append(NotificationInputs, &KafkaInput{})
  12. }
  13. type KafkaInput struct {
  14. topic string
  15. consumer sarama.Consumer
  16. messageChan chan *sarama.ConsumerMessage
  17. }
  18. func (k *KafkaInput) GetName() string {
  19. return "kafka"
  20. }
  21. func (k *KafkaInput) Initialize(configuration util.Configuration) error {
  22. glog.V(0).Infof("replication.notification.kafka.hosts: %v\n", configuration.GetStringSlice("hosts"))
  23. glog.V(0).Infof("replication.notification.kafka.topic: %v\n", configuration.GetString("topic"))
  24. return k.initialize(
  25. configuration.GetStringSlice("hosts"),
  26. configuration.GetString("topic"),
  27. )
  28. }
  29. func (k *KafkaInput) initialize(hosts []string, topic string) (err error) {
  30. config := sarama.NewConfig()
  31. config.Consumer.Return.Errors = true
  32. k.consumer, err = sarama.NewConsumer(hosts, config)
  33. if err != nil {
  34. panic(err)
  35. } else {
  36. glog.V(0).Infof("connected to %v", hosts)
  37. }
  38. k.topic = topic
  39. k.messageChan = make(chan *sarama.ConsumerMessage, 1)
  40. partitions, err := k.consumer.Partitions(topic)
  41. if err != nil {
  42. panic(err)
  43. }
  44. for _, partition := range partitions {
  45. partitionConsumer, err := k.consumer.ConsumePartition(topic, partition, sarama.OffsetNewest)
  46. if err != nil {
  47. panic(err)
  48. }
  49. go func() {
  50. for {
  51. select {
  52. case err := <-partitionConsumer.Errors():
  53. fmt.Println(err)
  54. case msg := <-partitionConsumer.Messages():
  55. k.messageChan <- msg
  56. }
  57. }
  58. }()
  59. }
  60. return nil
  61. }
  62. func (k *KafkaInput) ReceiveMessage() (key string, message *filer_pb.EventNotification, err error) {
  63. msg := <-k.messageChan
  64. key = string(msg.Key)
  65. message = &filer_pb.EventNotification{}
  66. err = proto.Unmarshal(msg.Value, message)
  67. return
  68. }