Chris Lu
6 years ago
7 changed files with 240 additions and 29 deletions
-
6weed/command/filer_replication.go
-
47weed/command/scaffold.go
-
2weed/notification/configuration.go
-
89weed/notification/google_pub_sub/google_pub_sub.go
-
6weed/replication/sink/gcssink/gcs_sink.go
-
109weed/replication/sub/notification_google_pub_sub.go
-
10weed/server/filer_server.go
@ -0,0 +1,89 @@ |
|||
package google_pub_sub |
|||
|
|||
import ( |
|||
"fmt" |
|||
"os" |
|||
|
|||
"github.com/chrislusf/seaweedfs/weed/glog" |
|||
"github.com/chrislusf/seaweedfs/weed/notification" |
|||
"github.com/chrislusf/seaweedfs/weed/util" |
|||
"github.com/golang/protobuf/proto" |
|||
"google.golang.org/api/option" |
|||
"context" |
|||
"cloud.google.com/go/pubsub" |
|||
) |
|||
|
|||
func init() { |
|||
notification.MessageQueues = append(notification.MessageQueues, &GooglePubSub{}) |
|||
} |
|||
|
|||
type GooglePubSub struct { |
|||
topic *pubsub.Topic |
|||
} |
|||
|
|||
func (k *GooglePubSub) GetName() string { |
|||
return "google_pub_sub" |
|||
} |
|||
|
|||
func (k *GooglePubSub) Initialize(configuration util.Configuration) (err error) { |
|||
glog.V(0).Infof("notification.google_pub_sub.project_id: %v", configuration.GetString("project_id")) |
|||
glog.V(0).Infof("notification.google_pub_sub.topic: %v", configuration.GetString("topic")) |
|||
return k.initialize( |
|||
configuration.GetString("google_application_credentials"), |
|||
configuration.GetString("project_id"), |
|||
configuration.GetString("topic"), |
|||
) |
|||
} |
|||
|
|||
func (k *GooglePubSub) initialize(google_application_credentials, projectId, topicName string) (err error) { |
|||
|
|||
ctx := context.Background() |
|||
// Creates a client.
|
|||
if google_application_credentials == "" { |
|||
var found bool |
|||
google_application_credentials, found = os.LookupEnv("GOOGLE_APPLICATION_CREDENTIALS") |
|||
if !found { |
|||
glog.Fatalf("need to specific GOOGLE_APPLICATION_CREDENTIALS env variable or google_application_credentials in filer.toml") |
|||
} |
|||
} |
|||
|
|||
client, err := pubsub.NewClient(ctx, projectId, option.WithCredentialsFile(google_application_credentials)) |
|||
if err != nil { |
|||
glog.Fatalf("Failed to create client: %v", err) |
|||
} |
|||
|
|||
k.topic = client.Topic(topicName) |
|||
if exists, err := k.topic.Exists(ctx); err == nil { |
|||
if !exists { |
|||
k.topic, err = client.CreateTopic(ctx, topicName) |
|||
if err != nil { |
|||
glog.Fatalf("Failed to create topic %s: %v", topicName, err) |
|||
} |
|||
} |
|||
} else { |
|||
glog.Fatalf("Failed to check topic %s: %v", topicName, err) |
|||
} |
|||
|
|||
return nil |
|||
} |
|||
|
|||
func (k *GooglePubSub) SendMessage(key string, message proto.Message) (err error) { |
|||
|
|||
bytes, err := proto.Marshal(message) |
|||
if err != nil { |
|||
return |
|||
} |
|||
|
|||
ctx := context.Background() |
|||
result := k.topic.Publish(ctx, &pubsub.Message{ |
|||
Data: bytes, |
|||
Attributes: map[string]string{"key": key}, |
|||
}) |
|||
|
|||
_, err = result.Get(ctx) |
|||
if err != nil { |
|||
return fmt.Errorf("send message to google pub sub %s: %v", k.topic.String(), err) |
|||
} |
|||
|
|||
return nil |
|||
} |
|||
@ -0,0 +1,109 @@ |
|||
package sub |
|||
|
|||
import ( |
|||
"context" |
|||
"fmt" |
|||
"os" |
|||
|
|||
"github.com/chrislusf/seaweedfs/weed/glog" |
|||
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb" |
|||
"github.com/chrislusf/seaweedfs/weed/util" |
|||
"github.com/golang/protobuf/proto" |
|||
"cloud.google.com/go/pubsub" |
|||
"google.golang.org/api/option" |
|||
) |
|||
|
|||
func init() { |
|||
NotificationInputs = append(NotificationInputs, &GooglePubSubInput{}) |
|||
} |
|||
|
|||
type GooglePubSubInput struct { |
|||
sub *pubsub.Subscription |
|||
topicName string |
|||
messageChan chan *pubsub.Message |
|||
} |
|||
|
|||
func (k *GooglePubSubInput) GetName() string { |
|||
return "google_pub_sub" |
|||
} |
|||
|
|||
func (k *GooglePubSubInput) Initialize(configuration util.Configuration) error { |
|||
glog.V(0).Infof("notification.google_pub_sub.project_id: %v", configuration.GetString("project_id")) |
|||
glog.V(0).Infof("notification.google_pub_sub.topic: %v", configuration.GetString("topic")) |
|||
return k.initialize( |
|||
configuration.GetString("google_application_credentials"), |
|||
configuration.GetString("project_id"), |
|||
configuration.GetString("topic"), |
|||
) |
|||
} |
|||
|
|||
func (k *GooglePubSubInput) initialize(google_application_credentials, projectId, topicName string) (err error) { |
|||
|
|||
ctx := context.Background() |
|||
// Creates a client.
|
|||
if google_application_credentials == "" { |
|||
var found bool |
|||
google_application_credentials, found = os.LookupEnv("GOOGLE_APPLICATION_CREDENTIALS") |
|||
if !found { |
|||
glog.Fatalf("need to specific GOOGLE_APPLICATION_CREDENTIALS env variable or google_application_credentials in filer.toml") |
|||
} |
|||
} |
|||
|
|||
client, err := pubsub.NewClient(ctx, projectId, option.WithCredentialsFile(google_application_credentials)) |
|||
if err != nil { |
|||
glog.Fatalf("Failed to create client: %v", err) |
|||
} |
|||
|
|||
k.topicName = topicName |
|||
topic := client.Topic(topicName) |
|||
if exists, err := topic.Exists(ctx); err == nil { |
|||
if !exists { |
|||
topic, err = client.CreateTopic(ctx, topicName) |
|||
if err != nil { |
|||
glog.Fatalf("Failed to create topic %s: %v", topicName, err) |
|||
} |
|||
} |
|||
} else { |
|||
glog.Fatalf("Failed to check topic %s: %v", topicName, err) |
|||
} |
|||
|
|||
subscriptionName := "seaweedfs_sub" |
|||
|
|||
k.sub = client.Subscription(subscriptionName) |
|||
if exists, err := k.sub.Exists(ctx); err == nil { |
|||
if !exists { |
|||
k.sub, err = client.CreateSubscription(ctx, subscriptionName, pubsub.SubscriptionConfig{Topic: topic}) |
|||
if err != nil { |
|||
glog.Fatalf("Failed to create subscription %s: %v", subscriptionName, err) |
|||
} |
|||
} |
|||
} else { |
|||
glog.Fatalf("Failed to check subscription %s: %v", topicName, err) |
|||
} |
|||
|
|||
k.messageChan = make(chan *pubsub.Message, 1) |
|||
|
|||
go k.sub.Receive(ctx, func(ctx context.Context, m *pubsub.Message) { |
|||
k.messageChan <- m |
|||
m.Ack() |
|||
}) |
|||
|
|||
return err |
|||
} |
|||
|
|||
func (k *GooglePubSubInput) ReceiveMessage() (key string, message *filer_pb.EventNotification, err error) { |
|||
|
|||
m := <-k.messageChan |
|||
|
|||
// process the message
|
|||
key = m.Attributes["key"] |
|||
message = &filer_pb.EventNotification{} |
|||
err = proto.Unmarshal(m.Data, message) |
|||
|
|||
if err != nil { |
|||
err = fmt.Errorf("unmarshal message from google pubsub %s: %v", k.topicName, err) |
|||
return |
|||
} |
|||
|
|||
return |
|||
} |
|||
Write
Preview
Loading…
Cancel
Save
Reference in new issue