Browse Source
Merge remote-tracking branch 'origin/master' into filer_mongodb
Merge remote-tracking branch 'origin/master' into filer_mongodb
# Conflicts: # go.mod # go.sum # weed/server/filer_server.gopull/1280/head
bukton
5 years ago
101 changed files with 3018 additions and 1122 deletions
-
6README.md
-
12go.mod
-
17go.sum
-
2k8s/seaweedfs/Chart.yaml
-
2k8s/seaweedfs/values.yaml
-
19other/java/client/src/main/proto/filer.proto
-
75unmaintained/see_log_entry/see_log_entry.go
-
1weed/command/filer_replication.go
-
3weed/command/master.go
-
12weed/command/mount.go
-
6weed/command/mount_std.go
-
28weed/command/msg_broker.go
-
17weed/command/scaffold.go
-
15weed/command/shell.go
-
3weed/command/volume.go
-
2weed/command/watch.go
-
7weed/command/webdav.go
-
1weed/filer2/entry.go
-
6weed/filer2/entry_codec.go
-
37weed/filer2/filechunks.go
-
14weed/filer2/filer.go
-
19weed/filer2/filer_buckets.go
-
24weed/filer2/filer_notify.go
-
47weed/filer2/filer_notify_append.go
-
4weed/filer2/leveldb/leveldb_store_test.go
-
4weed/filer2/leveldb2/leveldb2_store_test.go
-
14weed/filer2/reader_at.go
-
42weed/filer2/redis2/redis_cluster_store.go
-
36weed/filer2/redis2/redis_store.go
-
162weed/filer2/redis2/universal_redis_store.go
-
4weed/filer2/stream.go
-
6weed/filer2/topics.go
-
4weed/filesys/dir.go
-
18weed/filesys/dirty_page.go
-
30weed/filesys/filehandle.go
-
32weed/filesys/fscache.go
-
72weed/filesys/wfs.go
-
2weed/images/orientation.go
-
113weed/messaging/broker/broker_append.go
-
15weed/messaging/broker/broker_grpc_server.go
-
99weed/messaging/broker/broker_grpc_server_publish.go
-
88weed/messaging/broker/broker_grpc_server_subscribe.go
-
11weed/messaging/broker/broker_server.go
-
80weed/messaging/broker/topic_lock.go
-
12weed/operation/upload_content.go
-
2weed/pb/Makefile
-
19weed/pb/filer.proto
-
448weed/pb/filer_pb/filer.pb.go
-
110weed/pb/messaging.proto
-
824weed/pb/messaging_pb/messaging.pb.go
-
66weed/pb/queue.proto
-
516weed/pb/queue_pb/queue.pb.go
-
5weed/pb/shared_values.go
-
2weed/replication/sink/azuresink/azure_sink.go
-
2weed/replication/sink/b2sink/b2_sink.go
-
4weed/replication/sink/filersink/filer_sink.go
-
2weed/replication/sink/gcssink/gcs_sink.go
-
9weed/replication/sink/s3sink/s3_sink.go
-
4weed/replication/sub/notification_aws_sqs.go
-
4weed/s3api/filer_multipart.go
-
2weed/s3api/s3api_objects_list_handlers.go
-
35weed/server/filer_grpc_server.go
-
8weed/server/filer_grpc_server_listen.go
-
28weed/server/filer_grpc_server_rename.go
-
12weed/server/filer_server.go
-
21weed/server/filer_server_handlers_read.go
-
42weed/server/filer_server_handlers_write.go
-
16weed/server/filer_server_handlers_write_autochunk.go
-
4weed/server/filer_server_handlers_write_cipher.go
-
14weed/server/master_grpc_server.go
-
11weed/server/master_server.go
-
23weed/server/msg_broker_grpc_server.go
-
2weed/server/volume_grpc_tail.go
-
8weed/server/volume_server_handlers_write.go
-
10weed/server/volume_server_ui/templates.go
-
13weed/server/webdav_server.go
-
2weed/shell/commands.go
-
4weed/storage/backend/memory_map/memory_map_backend.go
-
7weed/storage/backend/volume_create.go
-
7weed/storage/backend/volume_create_linux.go
-
7weed/storage/backend/volume_create_windows.go
-
6weed/storage/store.go
-
2weed/storage/volume_loading.go
-
7weed/storage/volume_read_write.go
-
4weed/storage/volume_vacuum.go
-
2weed/storage/volume_vacuum_test.go
-
10weed/topology/store_replicate.go
-
7weed/util/bytes.go
-
113weed/util/chunk_cache/chunk_cache.go
-
12weed/util/chunk_cache/chunk_cache_in_memory.go
-
145weed/util/chunk_cache/chunk_cache_on_disk.go
-
59weed/util/chunk_cache/chunk_cache_on_disk_test.go
-
89weed/util/chunk_cache/on_disk_cache_layer.go
-
3weed/util/config.go
-
2weed/util/constants.go
-
2weed/util/http_util.go
-
54weed/util/log_buffer/log_buffer.go
-
40weed/util/log_buffer/sealed_buffer.go
-
25weed/util/network.go
-
16weed/util/parse.go
@ -1,4 +1,4 @@ |
|||
apiVersion: v1 |
|||
description: SeaweedFS |
|||
name: seaweedfs |
|||
version: 1.71 |
|||
version: 1.74 |
|||
@ -0,0 +1,75 @@ |
|||
package main |
|||
|
|||
import ( |
|||
"flag" |
|||
"fmt" |
|||
"io" |
|||
"log" |
|||
"os" |
|||
|
|||
"github.com/golang/protobuf/proto" |
|||
|
|||
"github.com/chrislusf/seaweedfs/weed/filer2" |
|||
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb" |
|||
"github.com/chrislusf/seaweedfs/weed/util" |
|||
) |
|||
|
|||
var ( |
|||
logdataFile = flag.String("logdata", "", "log data file saved under "+ filer2.SystemLogDir) |
|||
) |
|||
|
|||
func main() { |
|||
flag.Parse() |
|||
|
|||
dst, err := os.OpenFile(*logdataFile, os.O_RDONLY, 0644) |
|||
if err != nil { |
|||
log.Fatalf("failed to open %s: %v", *logdataFile, err) |
|||
} |
|||
defer dst.Close() |
|||
|
|||
err = walkLogEntryFile(dst) |
|||
if err != nil { |
|||
log.Fatalf("failed to visit %s: %v", *logdataFile, err) |
|||
} |
|||
|
|||
} |
|||
|
|||
func walkLogEntryFile(dst *os.File) error { |
|||
|
|||
sizeBuf := make([]byte, 4) |
|||
|
|||
for { |
|||
if n, err := dst.Read(sizeBuf); n != 4 { |
|||
if err == io.EOF { |
|||
return nil |
|||
} |
|||
return err |
|||
} |
|||
|
|||
size := util.BytesToUint32(sizeBuf) |
|||
|
|||
data := make([]byte, int(size)) |
|||
|
|||
if n, err := dst.Read(data); n != len(data) { |
|||
return err |
|||
} |
|||
|
|||
logEntry := &filer_pb.LogEntry{} |
|||
err := proto.Unmarshal(data, logEntry) |
|||
if err != nil { |
|||
log.Printf("unexpected unmarshal filer_pb.LogEntry: %v", err) |
|||
return nil |
|||
} |
|||
|
|||
event := &filer_pb.SubscribeMetadataResponse{} |
|||
err = proto.Unmarshal(logEntry.Data, event) |
|||
if err != nil { |
|||
log.Printf("unexpected unmarshal filer_pb.SubscribeMetadataResponse: %v", err) |
|||
return nil |
|||
} |
|||
|
|||
fmt.Printf("event: %+v\n", event) |
|||
|
|||
} |
|||
|
|||
} |
|||
@ -0,0 +1,42 @@ |
|||
package redis2 |
|||
|
|||
import ( |
|||
"github.com/chrislusf/seaweedfs/weed/filer2" |
|||
"github.com/chrislusf/seaweedfs/weed/util" |
|||
"github.com/go-redis/redis" |
|||
) |
|||
|
|||
func init() { |
|||
filer2.Stores = append(filer2.Stores, &RedisCluster2Store{}) |
|||
} |
|||
|
|||
type RedisCluster2Store struct { |
|||
UniversalRedis2Store |
|||
} |
|||
|
|||
func (store *RedisCluster2Store) GetName() string { |
|||
return "redis_cluster2" |
|||
} |
|||
|
|||
func (store *RedisCluster2Store) Initialize(configuration util.Configuration, prefix string) (err error) { |
|||
|
|||
configuration.SetDefault(prefix+"useReadOnly", true) |
|||
configuration.SetDefault(prefix+"routeByLatency", true) |
|||
|
|||
return store.initialize( |
|||
configuration.GetStringSlice(prefix+"addresses"), |
|||
configuration.GetString(prefix+"password"), |
|||
configuration.GetBool(prefix+"useReadOnly"), |
|||
configuration.GetBool(prefix+"routeByLatency"), |
|||
) |
|||
} |
|||
|
|||
func (store *RedisCluster2Store) initialize(addresses []string, password string, readOnly, routeByLatency bool) (err error) { |
|||
store.Client = redis.NewClusterClient(&redis.ClusterOptions{ |
|||
Addrs: addresses, |
|||
Password: password, |
|||
ReadOnly: readOnly, |
|||
RouteByLatency: routeByLatency, |
|||
}) |
|||
return |
|||
} |
|||
@ -0,0 +1,36 @@ |
|||
package redis2 |
|||
|
|||
import ( |
|||
"github.com/chrislusf/seaweedfs/weed/filer2" |
|||
"github.com/chrislusf/seaweedfs/weed/util" |
|||
"github.com/go-redis/redis" |
|||
) |
|||
|
|||
func init() { |
|||
filer2.Stores = append(filer2.Stores, &Redis2Store{}) |
|||
} |
|||
|
|||
type Redis2Store struct { |
|||
UniversalRedis2Store |
|||
} |
|||
|
|||
func (store *Redis2Store) GetName() string { |
|||
return "redis2" |
|||
} |
|||
|
|||
func (store *Redis2Store) Initialize(configuration util.Configuration, prefix string) (err error) { |
|||
return store.initialize( |
|||
configuration.GetString(prefix+"address"), |
|||
configuration.GetString(prefix+"password"), |
|||
configuration.GetInt(prefix+"database"), |
|||
) |
|||
} |
|||
|
|||
func (store *Redis2Store) initialize(hostPort string, password string, database int) (err error) { |
|||
store.Client = redis.NewClient(&redis.Options{ |
|||
Addr: hostPort, |
|||
Password: password, |
|||
DB: database, |
|||
}) |
|||
return |
|||
} |
|||
@ -0,0 +1,162 @@ |
|||
package redis2 |
|||
|
|||
import ( |
|||
"context" |
|||
"fmt" |
|||
"time" |
|||
|
|||
"github.com/go-redis/redis" |
|||
|
|||
"github.com/chrislusf/seaweedfs/weed/filer2" |
|||
"github.com/chrislusf/seaweedfs/weed/glog" |
|||
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb" |
|||
"github.com/chrislusf/seaweedfs/weed/util" |
|||
) |
|||
|
|||
const ( |
|||
DIR_LIST_MARKER = "\x00" |
|||
) |
|||
|
|||
type UniversalRedis2Store struct { |
|||
Client redis.UniversalClient |
|||
} |
|||
|
|||
func (store *UniversalRedis2Store) BeginTransaction(ctx context.Context) (context.Context, error) { |
|||
return ctx, nil |
|||
} |
|||
func (store *UniversalRedis2Store) CommitTransaction(ctx context.Context) error { |
|||
return nil |
|||
} |
|||
func (store *UniversalRedis2Store) RollbackTransaction(ctx context.Context) error { |
|||
return nil |
|||
} |
|||
|
|||
func (store *UniversalRedis2Store) InsertEntry(ctx context.Context, entry *filer2.Entry) (err error) { |
|||
|
|||
value, err := entry.EncodeAttributesAndChunks() |
|||
if err != nil { |
|||
return fmt.Errorf("encoding %s %+v: %v", entry.FullPath, entry.Attr, err) |
|||
} |
|||
|
|||
if err = store.Client.Set(string(entry.FullPath), value, time.Duration(entry.TtlSec)*time.Second).Err(); err != nil { |
|||
return fmt.Errorf("persisting %s : %v", entry.FullPath, err) |
|||
} |
|||
|
|||
dir, name := entry.FullPath.DirAndName() |
|||
if name != "" { |
|||
if err = store.Client.ZAddNX(genDirectoryListKey(dir), redis.Z{Score: 0, Member: name}).Err(); err != nil { |
|||
return fmt.Errorf("persisting %s in parent dir: %v", entry.FullPath, err) |
|||
} |
|||
} |
|||
|
|||
return nil |
|||
} |
|||
|
|||
func (store *UniversalRedis2Store) UpdateEntry(ctx context.Context, entry *filer2.Entry) (err error) { |
|||
|
|||
return store.InsertEntry(ctx, entry) |
|||
} |
|||
|
|||
func (store *UniversalRedis2Store) FindEntry(ctx context.Context, fullpath util.FullPath) (entry *filer2.Entry, err error) { |
|||
|
|||
data, err := store.Client.Get(string(fullpath)).Result() |
|||
if err == redis.Nil { |
|||
return nil, filer_pb.ErrNotFound |
|||
} |
|||
|
|||
if err != nil { |
|||
return nil, fmt.Errorf("get %s : %v", fullpath, err) |
|||
} |
|||
|
|||
entry = &filer2.Entry{ |
|||
FullPath: fullpath, |
|||
} |
|||
err = entry.DecodeAttributesAndChunks([]byte(data)) |
|||
if err != nil { |
|||
return entry, fmt.Errorf("decode %s : %v", entry.FullPath, err) |
|||
} |
|||
|
|||
return entry, nil |
|||
} |
|||
|
|||
func (store *UniversalRedis2Store) DeleteEntry(ctx context.Context, fullpath util.FullPath) (err error) { |
|||
|
|||
_, err = store.Client.Del(string(fullpath)).Result() |
|||
|
|||
if err != nil { |
|||
return fmt.Errorf("delete %s : %v", fullpath, err) |
|||
} |
|||
|
|||
dir, name := fullpath.DirAndName() |
|||
if name != "" { |
|||
_, err = store.Client.ZRem(genDirectoryListKey(dir), name).Result() |
|||
if err != nil { |
|||
return fmt.Errorf("delete %s in parent dir: %v", fullpath, err) |
|||
} |
|||
} |
|||
|
|||
return nil |
|||
} |
|||
|
|||
func (store *UniversalRedis2Store) DeleteFolderChildren(ctx context.Context, fullpath util.FullPath) (err error) { |
|||
|
|||
members, err := store.Client.ZRange(genDirectoryListKey(string(fullpath)), 0, -1).Result() |
|||
if err != nil { |
|||
return fmt.Errorf("delete folder %s : %v", fullpath, err) |
|||
} |
|||
|
|||
for _, fileName := range members { |
|||
path := util.NewFullPath(string(fullpath), fileName) |
|||
_, err = store.Client.Del(string(path)).Result() |
|||
if err != nil { |
|||
return fmt.Errorf("delete %s in parent dir: %v", fullpath, err) |
|||
} |
|||
} |
|||
|
|||
return nil |
|||
} |
|||
|
|||
func (store *UniversalRedis2Store) ListDirectoryEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool, |
|||
limit int) (entries []*filer2.Entry, err error) { |
|||
|
|||
dirListKey := genDirectoryListKey(string(fullpath)) |
|||
start := int64(0) |
|||
if startFileName != "" { |
|||
start, _ = store.Client.ZRank(dirListKey, startFileName).Result() |
|||
if !inclusive { |
|||
start++ |
|||
} |
|||
} |
|||
members, err := store.Client.ZRange(dirListKey, start, start+int64(limit)-1).Result() |
|||
if err != nil { |
|||
return nil, fmt.Errorf("list %s : %v", fullpath, err) |
|||
} |
|||
|
|||
// fetch entry meta
|
|||
for _, fileName := range members { |
|||
path := util.NewFullPath(string(fullpath), fileName) |
|||
entry, err := store.FindEntry(ctx, path) |
|||
if err != nil { |
|||
glog.V(0).Infof("list %s : %v", path, err) |
|||
} else { |
|||
if entry.TtlSec > 0 { |
|||
if entry.Attr.Crtime.Add(time.Duration(entry.TtlSec) * time.Second).Before(time.Now()) { |
|||
store.Client.Del(string(path)).Result() |
|||
store.Client.ZRem(dirListKey, fileName).Result() |
|||
continue |
|||
} |
|||
} |
|||
entries = append(entries, entry) |
|||
} |
|||
} |
|||
|
|||
return entries, err |
|||
} |
|||
|
|||
func genDirectoryListKey(dir string) (dirList string) { |
|||
return dir + DIR_LIST_MARKER |
|||
} |
|||
|
|||
func (store *UniversalRedis2Store) Shutdown() { |
|||
store.Client.Close() |
|||
} |
|||
@ -0,0 +1,6 @@ |
|||
package filer2 |
|||
|
|||
const ( |
|||
TopicsDir = "/topics" |
|||
SystemLogDir = TopicsDir + "/.system/log" |
|||
) |
|||
@ -0,0 +1,113 @@ |
|||
package broker |
|||
|
|||
import ( |
|||
"context" |
|||
"fmt" |
|||
"time" |
|||
|
|||
"github.com/chrislusf/seaweedfs/weed/glog" |
|||
"github.com/chrislusf/seaweedfs/weed/operation" |
|||
"github.com/chrislusf/seaweedfs/weed/pb" |
|||
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb" |
|||
"github.com/chrislusf/seaweedfs/weed/pb/messaging_pb" |
|||
"github.com/chrislusf/seaweedfs/weed/security" |
|||
"github.com/chrislusf/seaweedfs/weed/util" |
|||
) |
|||
|
|||
func (broker *MessageBroker) appendToFile(targetFile string, topicConfig *messaging_pb.TopicConfiguration, data []byte) error { |
|||
|
|||
assignResult, uploadResult, err2 := broker.assignAndUpload(topicConfig, data) |
|||
if err2 != nil { |
|||
return err2 |
|||
} |
|||
|
|||
dir, name := util.FullPath(targetFile).DirAndName() |
|||
|
|||
chunk := &filer_pb.FileChunk{ |
|||
FileId: assignResult.Fid, |
|||
Offset: 0, // needs to be fixed during appending
|
|||
Size: uint64(uploadResult.Size), |
|||
Mtime: time.Now().UnixNano(), |
|||
ETag: uploadResult.ETag, |
|||
IsGzipped: uploadResult.Gzip > 0, |
|||
} |
|||
|
|||
// append the chunk
|
|||
if err := broker.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error { |
|||
|
|||
request := &filer_pb.AppendToEntryRequest{ |
|||
Directory: dir, |
|||
EntryName: name, |
|||
Chunks: []*filer_pb.FileChunk{chunk}, |
|||
} |
|||
|
|||
_, err := client.AppendToEntry(context.Background(), request) |
|||
if err != nil { |
|||
glog.V(0).Infof("append to file %v: %v", request, err) |
|||
return err |
|||
} |
|||
|
|||
return nil |
|||
}); err != nil { |
|||
return fmt.Errorf("append to file %v: %v", targetFile, err) |
|||
} |
|||
|
|||
return nil |
|||
} |
|||
|
|||
func (broker *MessageBroker) assignAndUpload(topicConfig *messaging_pb.TopicConfiguration, data []byte) (*operation.AssignResult, *operation.UploadResult, error) { |
|||
|
|||
var assignResult = &operation.AssignResult{} |
|||
|
|||
// assign a volume location
|
|||
if err := broker.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error { |
|||
|
|||
request := &filer_pb.AssignVolumeRequest{ |
|||
Count: 1, |
|||
Replication: topicConfig.Replication, |
|||
Collection: topicConfig.Collection, |
|||
} |
|||
|
|||
resp, err := client.AssignVolume(context.Background(), request) |
|||
if err != nil { |
|||
glog.V(0).Infof("assign volume failure %v: %v", request, err) |
|||
return err |
|||
} |
|||
if resp.Error != "" { |
|||
return fmt.Errorf("assign volume failure %v: %v", request, resp.Error) |
|||
} |
|||
|
|||
assignResult.Auth = security.EncodedJwt(resp.Auth) |
|||
assignResult.Fid = resp.FileId |
|||
assignResult.Url = resp.Url |
|||
assignResult.PublicUrl = resp.PublicUrl |
|||
assignResult.Count = uint64(resp.Count) |
|||
|
|||
return nil |
|||
}); err != nil { |
|||
return nil, nil, err |
|||
} |
|||
|
|||
// upload data
|
|||
targetUrl := fmt.Sprintf("http://%s/%s", assignResult.Url, assignResult.Fid) |
|||
uploadResult, err := operation.UploadData(targetUrl, "", broker.option.Cipher, data, false, "", nil, assignResult.Auth) |
|||
if err != nil { |
|||
return nil, nil, fmt.Errorf("upload data %s: %v", targetUrl, err) |
|||
} |
|||
// println("uploaded to", targetUrl)
|
|||
return assignResult, uploadResult, nil |
|||
} |
|||
|
|||
func (broker *MessageBroker) WithFilerClient(fn func(filer_pb.SeaweedFilerClient) error) (err error) { |
|||
|
|||
for _, filer := range broker.option.Filers { |
|||
if err = pb.WithFilerClient(filer, broker.grpcDialOption, fn); err != nil { |
|||
glog.V(0).Infof("fail to connect to %s: %v", filer, err) |
|||
} else { |
|||
break |
|||
} |
|||
} |
|||
|
|||
return |
|||
|
|||
} |
|||
@ -0,0 +1,15 @@ |
|||
package broker |
|||
|
|||
import ( |
|||
"context" |
|||
|
|||
"github.com/chrislusf/seaweedfs/weed/pb/messaging_pb" |
|||
) |
|||
|
|||
func (broker *MessageBroker) ConfigureTopic(c context.Context, request *messaging_pb.ConfigureTopicRequest) (*messaging_pb.ConfigureTopicResponse, error) { |
|||
panic("implement me") |
|||
} |
|||
|
|||
func (broker *MessageBroker) GetTopicConfiguration(c context.Context, request *messaging_pb.GetTopicConfigurationRequest) (*messaging_pb.GetTopicConfigurationResponse, error) { |
|||
panic("implement me") |
|||
} |
|||
@ -0,0 +1,99 @@ |
|||
package broker |
|||
|
|||
import ( |
|||
"fmt" |
|||
"io" |
|||
"time" |
|||
|
|||
"github.com/golang/protobuf/proto" |
|||
|
|||
"github.com/chrislusf/seaweedfs/weed/filer2" |
|||
"github.com/chrislusf/seaweedfs/weed/glog" |
|||
"github.com/chrislusf/seaweedfs/weed/pb/messaging_pb" |
|||
) |
|||
|
|||
func (broker *MessageBroker) Publish(stream messaging_pb.SeaweedMessaging_PublishServer) error { |
|||
|
|||
// process initial request
|
|||
in, err := stream.Recv() |
|||
if err == io.EOF { |
|||
return nil |
|||
} |
|||
if err != nil { |
|||
return err |
|||
} |
|||
|
|||
// TODO look it up
|
|||
topicConfig := &messaging_pb.TopicConfiguration{ |
|||
|
|||
} |
|||
|
|||
// get lock
|
|||
tp := TopicPartition{ |
|||
Namespace: in.Init.Namespace, |
|||
Topic: in.Init.Topic, |
|||
Partition: in.Init.Partition, |
|||
} |
|||
logBuffer := broker.topicLocks.RequestPublisherLock(tp, func(startTime, stopTime time.Time, buf []byte) { |
|||
|
|||
targetFile := fmt.Sprintf( |
|||
"%s/%s/%s/%04d-%02d-%02d/%02d-%02d.part%02d", |
|||
filer2.TopicsDir, tp.Namespace, tp.Topic, |
|||
startTime.Year(), startTime.Month(), startTime.Day(), startTime.Hour(), startTime.Minute(), |
|||
tp.Partition, |
|||
) |
|||
|
|||
if err := broker.appendToFile(targetFile, topicConfig, buf); err != nil { |
|||
glog.V(0).Infof("log write failed %s: %v", targetFile, err) |
|||
} |
|||
|
|||
}) |
|||
defer broker.topicLocks.ReleaseLock(tp, true) |
|||
|
|||
updatesChan := make(chan int32) |
|||
|
|||
go func() { |
|||
for update := range updatesChan { |
|||
if err := stream.Send(&messaging_pb.PublishResponse{ |
|||
Config: &messaging_pb.PublishResponse_ConfigMessage{ |
|||
PartitionCount: update, |
|||
}, |
|||
}); err != nil { |
|||
glog.V(0).Infof("err sending publish response: %v", err) |
|||
return |
|||
} |
|||
} |
|||
}() |
|||
|
|||
|
|||
// process each message
|
|||
for { |
|||
in, err := stream.Recv() |
|||
if err == io.EOF { |
|||
return nil |
|||
} |
|||
if err != nil { |
|||
return err |
|||
} |
|||
|
|||
if in.Data == nil { |
|||
continue |
|||
} |
|||
|
|||
m := &messaging_pb.Message{ |
|||
Timestamp: time.Now().UnixNano(), |
|||
Key: in.Data.Key, |
|||
Value: in.Data.Value, |
|||
Headers: in.Data.Headers, |
|||
} |
|||
|
|||
data, err := proto.Marshal(m) |
|||
if err != nil { |
|||
glog.Errorf("marshall error: %v\n", err) |
|||
continue |
|||
} |
|||
|
|||
logBuffer.AddToBuffer(in.Data.Key, data) |
|||
|
|||
} |
|||
} |
|||
@ -0,0 +1,88 @@ |
|||
package broker |
|||
|
|||
import ( |
|||
"io" |
|||
"sync" |
|||
"time" |
|||
|
|||
"github.com/golang/protobuf/proto" |
|||
|
|||
"github.com/chrislusf/seaweedfs/weed/glog" |
|||
"github.com/chrislusf/seaweedfs/weed/pb/messaging_pb" |
|||
"github.com/chrislusf/seaweedfs/weed/util" |
|||
) |
|||
|
|||
func (broker *MessageBroker) Subscribe(stream messaging_pb.SeaweedMessaging_SubscribeServer) error { |
|||
|
|||
// process initial request
|
|||
in, err := stream.Recv() |
|||
if err == io.EOF { |
|||
return nil |
|||
} |
|||
if err != nil { |
|||
return err |
|||
} |
|||
|
|||
subscriberId := in.Init.SubscriberId |
|||
|
|||
// get lock
|
|||
tp := TopicPartition{ |
|||
Namespace: in.Init.Namespace, |
|||
Topic: in.Init.Topic, |
|||
Partition: in.Init.Partition, |
|||
} |
|||
lock := broker.topicLocks.RequestSubscriberLock(tp) |
|||
defer broker.topicLocks.ReleaseLock(tp, false) |
|||
cond := sync.NewCond(&lock.Mutex) |
|||
|
|||
lastReadTime := time.Now() |
|||
switch in.Init.StartPosition { |
|||
case messaging_pb.SubscriberMessage_InitMessage_TIMESTAMP: |
|||
lastReadTime = time.Unix(0, in.Init.TimestampNs) |
|||
case messaging_pb.SubscriberMessage_InitMessage_LATEST: |
|||
case messaging_pb.SubscriberMessage_InitMessage_EARLIEST: |
|||
} |
|||
|
|||
// how to process each message
|
|||
// an error returned will end the subscription
|
|||
eachMessageFn := func(m *messaging_pb.Message) error { |
|||
err := stream.Send(&messaging_pb.BrokerMessage{ |
|||
Data: m, |
|||
}) |
|||
if err != nil { |
|||
glog.V(0).Infof("=> subscriber %v: %+v", subscriberId, err) |
|||
} |
|||
return err |
|||
} |
|||
|
|||
// loop through all messages
|
|||
for { |
|||
|
|||
_, buf := lock.logBuffer.ReadFromBuffer(lastReadTime) |
|||
|
|||
for pos := 0; pos+4 < len(buf); { |
|||
|
|||
size := util.BytesToUint32(buf[pos : pos+4]) |
|||
entryData := buf[pos+4 : pos+4+int(size)] |
|||
|
|||
m := &messaging_pb.Message{} |
|||
if err = proto.Unmarshal(entryData, m); err != nil { |
|||
glog.Errorf("unexpected unmarshal messaging_pb.Message: %v", err) |
|||
pos += 4 + int(size) |
|||
continue |
|||
} |
|||
|
|||
if err = eachMessageFn(m); err != nil { |
|||
return err |
|||
} |
|||
|
|||
lastReadTime = time.Unix(0, m.Timestamp) |
|||
pos += 4 + int(size) |
|||
} |
|||
|
|||
lock.Mutex.Lock() |
|||
cond.Wait() |
|||
lock.Mutex.Unlock() |
|||
} |
|||
|
|||
} |
|||
@ -0,0 +1,80 @@ |
|||
package broker |
|||
|
|||
import ( |
|||
"sync" |
|||
"time" |
|||
|
|||
"github.com/chrislusf/seaweedfs/weed/util/log_buffer" |
|||
) |
|||
|
|||
type TopicPartition struct { |
|||
Namespace string |
|||
Topic string |
|||
Partition int32 |
|||
} |
|||
type TopicLock struct { |
|||
sync.Mutex |
|||
subscriberCount int |
|||
publisherCount int |
|||
logBuffer *log_buffer.LogBuffer |
|||
} |
|||
|
|||
type TopicLocks struct { |
|||
sync.Mutex |
|||
locks map[TopicPartition]*TopicLock |
|||
} |
|||
|
|||
func NewTopicLocks() *TopicLocks { |
|||
return &TopicLocks{ |
|||
locks: make(map[TopicPartition]*TopicLock), |
|||
} |
|||
} |
|||
|
|||
func (tl *TopicLocks) RequestSubscriberLock(partition TopicPartition) *TopicLock { |
|||
tl.Lock() |
|||
defer tl.Unlock() |
|||
|
|||
lock, found := tl.locks[partition] |
|||
if !found { |
|||
lock = &TopicLock{} |
|||
tl.locks[partition] = lock |
|||
} |
|||
lock.subscriberCount++ |
|||
|
|||
return lock |
|||
} |
|||
|
|||
func (tl *TopicLocks) RequestPublisherLock(partition TopicPartition, flushFn func(startTime, stopTime time.Time, buf []byte)) *log_buffer.LogBuffer { |
|||
tl.Lock() |
|||
defer tl.Unlock() |
|||
|
|||
lock, found := tl.locks[partition] |
|||
if !found { |
|||
lock = &TopicLock{} |
|||
tl.locks[partition] = lock |
|||
} |
|||
lock.publisherCount++ |
|||
cond := sync.NewCond(&lock.Mutex) |
|||
lock.logBuffer = log_buffer.NewLogBuffer(time.Minute, flushFn, func() { |
|||
cond.Broadcast() |
|||
}) |
|||
return lock.logBuffer |
|||
} |
|||
|
|||
func (tl *TopicLocks) ReleaseLock(partition TopicPartition, isPublisher bool) { |
|||
tl.Lock() |
|||
defer tl.Unlock() |
|||
|
|||
lock, found := tl.locks[partition] |
|||
if !found { |
|||
return |
|||
} |
|||
if isPublisher { |
|||
lock.publisherCount-- |
|||
} else { |
|||
lock.subscriberCount-- |
|||
} |
|||
if lock.subscriberCount <= 0 && lock.publisherCount <= 0 { |
|||
delete(tl.locks, partition) |
|||
} |
|||
} |
|||
@ -0,0 +1,110 @@ |
|||
syntax = "proto3"; |
|||
|
|||
package messaging_pb; |
|||
|
|||
option java_package = "seaweedfs.client"; |
|||
option java_outer_classname = "MessagingProto"; |
|||
|
|||
////////////////////////////////////////////////// |
|||
|
|||
service SeaweedMessaging { |
|||
|
|||
rpc Subscribe (stream SubscriberMessage) returns (stream BrokerMessage) { |
|||
} |
|||
|
|||
rpc Publish (stream PublishRequest) returns (stream PublishResponse) { |
|||
} |
|||
|
|||
rpc ConfigureTopic (ConfigureTopicRequest) returns (ConfigureTopicResponse) { |
|||
} |
|||
|
|||
rpc GetTopicConfiguration (GetTopicConfigurationRequest) returns (GetTopicConfigurationResponse) { |
|||
} |
|||
|
|||
} |
|||
|
|||
////////////////////////////////////////////////// |
|||
|
|||
message SubscriberMessage { |
|||
message InitMessage { |
|||
string namespace = 1; |
|||
string topic = 2; |
|||
int32 partition = 3; |
|||
enum StartPosition { |
|||
LATEST = 0; // Start at the newest message |
|||
EARLIEST = 1; // Start at the oldest message |
|||
TIMESTAMP = 2; // Start after a specified timestamp, exclusive |
|||
} |
|||
StartPosition startPosition = 4; // Where to begin consuming from |
|||
int64 timestampNs = 5; // timestamp in nano seconds |
|||
string subscriber_id = 6; // uniquely identify a subscriber to track consumption |
|||
} |
|||
InitMessage init = 1; |
|||
message AckMessage { |
|||
int64 message_id = 1; |
|||
} |
|||
AckMessage ack = 2; |
|||
} |
|||
|
|||
message Message { |
|||
int64 timestamp = 1 [jstype = JS_STRING]; // When the message was received by the broker |
|||
bytes key = 2; // Message key |
|||
bytes value = 3; // Message payload |
|||
map<string, bytes> headers = 4; // Message headers |
|||
} |
|||
|
|||
message BrokerMessage { |
|||
Message data = 1; |
|||
message RedirectMessage { |
|||
string new_broker = 1; |
|||
} |
|||
RedirectMessage redirect = 2; |
|||
} |
|||
|
|||
message PublishRequest { |
|||
message InitMessage { |
|||
string namespace = 1; // only needed on the initial request |
|||
string topic = 2; // only needed on the initial request |
|||
int32 partition = 3; |
|||
} |
|||
InitMessage init = 1; |
|||
message DataMessage { |
|||
bytes key = 1; // Message key |
|||
bytes value = 2; // Message payload |
|||
map<string, bytes> headers = 3; // Message headers |
|||
} |
|||
DataMessage data = 2; |
|||
} |
|||
|
|||
message PublishResponse { |
|||
message ConfigMessage { |
|||
int32 partition_count = 1; |
|||
} |
|||
ConfigMessage config = 1; |
|||
message RedirectMessage { |
|||
string new_broker = 1; |
|||
} |
|||
RedirectMessage redirect = 2; |
|||
} |
|||
|
|||
message ConfigureTopicRequest { |
|||
string namespace = 1; |
|||
string topic = 2; |
|||
TopicConfiguration configuration = 3; |
|||
} |
|||
message ConfigureTopicResponse { |
|||
} |
|||
|
|||
message GetTopicConfigurationRequest { |
|||
string namespace = 1; |
|||
string topic = 2; |
|||
} |
|||
message GetTopicConfigurationResponse { |
|||
TopicConfiguration configuration = 1; |
|||
} |
|||
|
|||
message TopicConfiguration { |
|||
int32 partition_count = 1; |
|||
string collection = 2; |
|||
string replication = 3; |
|||
} |
|||
@ -0,0 +1,824 @@ |
|||
// Code generated by protoc-gen-go.
|
|||
// source: messaging.proto
|
|||
// DO NOT EDIT!
|
|||
|
|||
/* |
|||
Package messaging_pb is a generated protocol buffer package. |
|||
|
|||
It is generated from these files: |
|||
messaging.proto |
|||
|
|||
It has these top-level messages: |
|||
SubscriberMessage |
|||
Message |
|||
BrokerMessage |
|||
PublishRequest |
|||
PublishResponse |
|||
ConfigureTopicRequest |
|||
ConfigureTopicResponse |
|||
GetTopicConfigurationRequest |
|||
GetTopicConfigurationResponse |
|||
TopicConfiguration |
|||
*/ |
|||
package messaging_pb |
|||
|
|||
import proto "github.com/golang/protobuf/proto" |
|||
import fmt "fmt" |
|||
import math "math" |
|||
|
|||
import ( |
|||
context "golang.org/x/net/context" |
|||
grpc "google.golang.org/grpc" |
|||
) |
|||
|
|||
// Reference imports to suppress errors if they are not otherwise used.
|
|||
var _ = proto.Marshal |
|||
var _ = fmt.Errorf |
|||
var _ = math.Inf |
|||
|
|||
// This is a compile-time assertion to ensure that this generated file
|
|||
// is compatible with the proto package it is being compiled against.
|
|||
// A compilation error at this line likely means your copy of the
|
|||
// proto package needs to be updated.
|
|||
const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package
|
|||
|
|||
type SubscriberMessage_InitMessage_StartPosition int32 |
|||
|
|||
const ( |
|||
SubscriberMessage_InitMessage_LATEST SubscriberMessage_InitMessage_StartPosition = 0 |
|||
SubscriberMessage_InitMessage_EARLIEST SubscriberMessage_InitMessage_StartPosition = 1 |
|||
SubscriberMessage_InitMessage_TIMESTAMP SubscriberMessage_InitMessage_StartPosition = 2 |
|||
) |
|||
|
|||
var SubscriberMessage_InitMessage_StartPosition_name = map[int32]string{ |
|||
0: "LATEST", |
|||
1: "EARLIEST", |
|||
2: "TIMESTAMP", |
|||
} |
|||
var SubscriberMessage_InitMessage_StartPosition_value = map[string]int32{ |
|||
"LATEST": 0, |
|||
"EARLIEST": 1, |
|||
"TIMESTAMP": 2, |
|||
} |
|||
|
|||
func (x SubscriberMessage_InitMessage_StartPosition) String() string { |
|||
return proto.EnumName(SubscriberMessage_InitMessage_StartPosition_name, int32(x)) |
|||
} |
|||
func (SubscriberMessage_InitMessage_StartPosition) EnumDescriptor() ([]byte, []int) { |
|||
return fileDescriptor0, []int{0, 0, 0} |
|||
} |
|||
|
|||
type SubscriberMessage struct { |
|||
Init *SubscriberMessage_InitMessage `protobuf:"bytes,1,opt,name=init" json:"init,omitempty"` |
|||
Ack *SubscriberMessage_AckMessage `protobuf:"bytes,2,opt,name=ack" json:"ack,omitempty"` |
|||
} |
|||
|
|||
func (m *SubscriberMessage) Reset() { *m = SubscriberMessage{} } |
|||
func (m *SubscriberMessage) String() string { return proto.CompactTextString(m) } |
|||
func (*SubscriberMessage) ProtoMessage() {} |
|||
func (*SubscriberMessage) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{0} } |
|||
|
|||
func (m *SubscriberMessage) GetInit() *SubscriberMessage_InitMessage { |
|||
if m != nil { |
|||
return m.Init |
|||
} |
|||
return nil |
|||
} |
|||
|
|||
func (m *SubscriberMessage) GetAck() *SubscriberMessage_AckMessage { |
|||
if m != nil { |
|||
return m.Ack |
|||
} |
|||
return nil |
|||
} |
|||
|
|||
type SubscriberMessage_InitMessage struct { |
|||
Namespace string `protobuf:"bytes,1,opt,name=namespace" json:"namespace,omitempty"` |
|||
Topic string `protobuf:"bytes,2,opt,name=topic" json:"topic,omitempty"` |
|||
Partition int32 `protobuf:"varint,3,opt,name=partition" json:"partition,omitempty"` |
|||
StartPosition SubscriberMessage_InitMessage_StartPosition `protobuf:"varint,4,opt,name=startPosition,enum=messaging_pb.SubscriberMessage_InitMessage_StartPosition" json:"startPosition,omitempty"` |
|||
TimestampNs int64 `protobuf:"varint,5,opt,name=timestampNs" json:"timestampNs,omitempty"` |
|||
SubscriberId string `protobuf:"bytes,6,opt,name=subscriber_id,json=subscriberId" json:"subscriber_id,omitempty"` |
|||
} |
|||
|
|||
func (m *SubscriberMessage_InitMessage) Reset() { *m = SubscriberMessage_InitMessage{} } |
|||
func (m *SubscriberMessage_InitMessage) String() string { return proto.CompactTextString(m) } |
|||
func (*SubscriberMessage_InitMessage) ProtoMessage() {} |
|||
func (*SubscriberMessage_InitMessage) Descriptor() ([]byte, []int) { |
|||
return fileDescriptor0, []int{0, 0} |
|||
} |
|||
|
|||
func (m *SubscriberMessage_InitMessage) GetNamespace() string { |
|||
if m != nil { |
|||
return m.Namespace |
|||
} |
|||
return "" |
|||
} |
|||
|
|||
func (m *SubscriberMessage_InitMessage) GetTopic() string { |
|||
if m != nil { |
|||
return m.Topic |
|||
} |
|||
return "" |
|||
} |
|||
|
|||
func (m *SubscriberMessage_InitMessage) GetPartition() int32 { |
|||
if m != nil { |
|||
return m.Partition |
|||
} |
|||
return 0 |
|||
} |
|||
|
|||
func (m *SubscriberMessage_InitMessage) GetStartPosition() SubscriberMessage_InitMessage_StartPosition { |
|||
if m != nil { |
|||
return m.StartPosition |
|||
} |
|||
return SubscriberMessage_InitMessage_LATEST |
|||
} |
|||
|
|||
func (m *SubscriberMessage_InitMessage) GetTimestampNs() int64 { |
|||
if m != nil { |
|||
return m.TimestampNs |
|||
} |
|||
return 0 |
|||
} |
|||
|
|||
func (m *SubscriberMessage_InitMessage) GetSubscriberId() string { |
|||
if m != nil { |
|||
return m.SubscriberId |
|||
} |
|||
return "" |
|||
} |
|||
|
|||
type SubscriberMessage_AckMessage struct { |
|||
MessageId int64 `protobuf:"varint,1,opt,name=message_id,json=messageId" json:"message_id,omitempty"` |
|||
} |
|||
|
|||
func (m *SubscriberMessage_AckMessage) Reset() { *m = SubscriberMessage_AckMessage{} } |
|||
func (m *SubscriberMessage_AckMessage) String() string { return proto.CompactTextString(m) } |
|||
func (*SubscriberMessage_AckMessage) ProtoMessage() {} |
|||
func (*SubscriberMessage_AckMessage) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{0, 1} } |
|||
|
|||
func (m *SubscriberMessage_AckMessage) GetMessageId() int64 { |
|||
if m != nil { |
|||
return m.MessageId |
|||
} |
|||
return 0 |
|||
} |
|||
|
|||
type Message struct { |
|||
Timestamp int64 `protobuf:"varint,1,opt,name=timestamp" json:"timestamp,omitempty"` |
|||
Key []byte `protobuf:"bytes,2,opt,name=key,proto3" json:"key,omitempty"` |
|||
Value []byte `protobuf:"bytes,3,opt,name=value,proto3" json:"value,omitempty"` |
|||
Headers map[string][]byte `protobuf:"bytes,4,rep,name=headers" json:"headers,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value,proto3"` |
|||
} |
|||
|
|||
func (m *Message) Reset() { *m = Message{} } |
|||
func (m *Message) String() string { return proto.CompactTextString(m) } |
|||
func (*Message) ProtoMessage() {} |
|||
func (*Message) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{1} } |
|||
|
|||
func (m *Message) GetTimestamp() int64 { |
|||
if m != nil { |
|||
return m.Timestamp |
|||
} |
|||
return 0 |
|||
} |
|||
|
|||
func (m *Message) GetKey() []byte { |
|||
if m != nil { |
|||
return m.Key |
|||
} |
|||
return nil |
|||
} |
|||
|
|||
func (m *Message) GetValue() []byte { |
|||
if m != nil { |
|||
return m.Value |
|||
} |
|||
return nil |
|||
} |
|||
|
|||
func (m *Message) GetHeaders() map[string][]byte { |
|||
if m != nil { |
|||
return m.Headers |
|||
} |
|||
return nil |
|||
} |
|||
|
|||
type BrokerMessage struct { |
|||
Data *Message `protobuf:"bytes,1,opt,name=data" json:"data,omitempty"` |
|||
Redirect *BrokerMessage_RedirectMessage `protobuf:"bytes,2,opt,name=redirect" json:"redirect,omitempty"` |
|||
} |
|||
|
|||
func (m *BrokerMessage) Reset() { *m = BrokerMessage{} } |
|||
func (m *BrokerMessage) String() string { return proto.CompactTextString(m) } |
|||
func (*BrokerMessage) ProtoMessage() {} |
|||
func (*BrokerMessage) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{2} } |
|||
|
|||
func (m *BrokerMessage) GetData() *Message { |
|||
if m != nil { |
|||
return m.Data |
|||
} |
|||
return nil |
|||
} |
|||
|
|||
func (m *BrokerMessage) GetRedirect() *BrokerMessage_RedirectMessage { |
|||
if m != nil { |
|||
return m.Redirect |
|||
} |
|||
return nil |
|||
} |
|||
|
|||
type BrokerMessage_RedirectMessage struct { |
|||
NewBroker string `protobuf:"bytes,1,opt,name=new_broker,json=newBroker" json:"new_broker,omitempty"` |
|||
} |
|||
|
|||
func (m *BrokerMessage_RedirectMessage) Reset() { *m = BrokerMessage_RedirectMessage{} } |
|||
func (m *BrokerMessage_RedirectMessage) String() string { return proto.CompactTextString(m) } |
|||
func (*BrokerMessage_RedirectMessage) ProtoMessage() {} |
|||
func (*BrokerMessage_RedirectMessage) Descriptor() ([]byte, []int) { |
|||
return fileDescriptor0, []int{2, 0} |
|||
} |
|||
|
|||
func (m *BrokerMessage_RedirectMessage) GetNewBroker() string { |
|||
if m != nil { |
|||
return m.NewBroker |
|||
} |
|||
return "" |
|||
} |
|||
|
|||
type PublishRequest struct { |
|||
Init *PublishRequest_InitMessage `protobuf:"bytes,1,opt,name=init" json:"init,omitempty"` |
|||
Data *PublishRequest_DataMessage `protobuf:"bytes,2,opt,name=data" json:"data,omitempty"` |
|||
} |
|||
|
|||
func (m *PublishRequest) Reset() { *m = PublishRequest{} } |
|||
func (m *PublishRequest) String() string { return proto.CompactTextString(m) } |
|||
func (*PublishRequest) ProtoMessage() {} |
|||
func (*PublishRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{3} } |
|||
|
|||
func (m *PublishRequest) GetInit() *PublishRequest_InitMessage { |
|||
if m != nil { |
|||
return m.Init |
|||
} |
|||
return nil |
|||
} |
|||
|
|||
func (m *PublishRequest) GetData() *PublishRequest_DataMessage { |
|||
if m != nil { |
|||
return m.Data |
|||
} |
|||
return nil |
|||
} |
|||
|
|||
type PublishRequest_InitMessage struct { |
|||
Namespace string `protobuf:"bytes,1,opt,name=namespace" json:"namespace,omitempty"` |
|||
Topic string `protobuf:"bytes,2,opt,name=topic" json:"topic,omitempty"` |
|||
Partition int32 `protobuf:"varint,3,opt,name=partition" json:"partition,omitempty"` |
|||
} |
|||
|
|||
func (m *PublishRequest_InitMessage) Reset() { *m = PublishRequest_InitMessage{} } |
|||
func (m *PublishRequest_InitMessage) String() string { return proto.CompactTextString(m) } |
|||
func (*PublishRequest_InitMessage) ProtoMessage() {} |
|||
func (*PublishRequest_InitMessage) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{3, 0} } |
|||
|
|||
func (m *PublishRequest_InitMessage) GetNamespace() string { |
|||
if m != nil { |
|||
return m.Namespace |
|||
} |
|||
return "" |
|||
} |
|||
|
|||
func (m *PublishRequest_InitMessage) GetTopic() string { |
|||
if m != nil { |
|||
return m.Topic |
|||
} |
|||
return "" |
|||
} |
|||
|
|||
func (m *PublishRequest_InitMessage) GetPartition() int32 { |
|||
if m != nil { |
|||
return m.Partition |
|||
} |
|||
return 0 |
|||
} |
|||
|
|||
type PublishRequest_DataMessage struct { |
|||
Key []byte `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"` |
|||
Value []byte `protobuf:"bytes,2,opt,name=value,proto3" json:"value,omitempty"` |
|||
Headers map[string][]byte `protobuf:"bytes,3,rep,name=headers" json:"headers,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value,proto3"` |
|||
} |
|||
|
|||
func (m *PublishRequest_DataMessage) Reset() { *m = PublishRequest_DataMessage{} } |
|||
func (m *PublishRequest_DataMessage) String() string { return proto.CompactTextString(m) } |
|||
func (*PublishRequest_DataMessage) ProtoMessage() {} |
|||
func (*PublishRequest_DataMessage) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{3, 1} } |
|||
|
|||
func (m *PublishRequest_DataMessage) GetKey() []byte { |
|||
if m != nil { |
|||
return m.Key |
|||
} |
|||
return nil |
|||
} |
|||
|
|||
func (m *PublishRequest_DataMessage) GetValue() []byte { |
|||
if m != nil { |
|||
return m.Value |
|||
} |
|||
return nil |
|||
} |
|||
|
|||
func (m *PublishRequest_DataMessage) GetHeaders() map[string][]byte { |
|||
if m != nil { |
|||
return m.Headers |
|||
} |
|||
return nil |
|||
} |
|||
|
|||
type PublishResponse struct { |
|||
Config *PublishResponse_ConfigMessage `protobuf:"bytes,1,opt,name=config" json:"config,omitempty"` |
|||
Redirect *PublishResponse_RedirectMessage `protobuf:"bytes,2,opt,name=redirect" json:"redirect,omitempty"` |
|||
} |
|||
|
|||
func (m *PublishResponse) Reset() { *m = PublishResponse{} } |
|||
func (m *PublishResponse) String() string { return proto.CompactTextString(m) } |
|||
func (*PublishResponse) ProtoMessage() {} |
|||
func (*PublishResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{4} } |
|||
|
|||
func (m *PublishResponse) GetConfig() *PublishResponse_ConfigMessage { |
|||
if m != nil { |
|||
return m.Config |
|||
} |
|||
return nil |
|||
} |
|||
|
|||
func (m *PublishResponse) GetRedirect() *PublishResponse_RedirectMessage { |
|||
if m != nil { |
|||
return m.Redirect |
|||
} |
|||
return nil |
|||
} |
|||
|
|||
type PublishResponse_ConfigMessage struct { |
|||
PartitionCount int32 `protobuf:"varint,1,opt,name=partition_count,json=partitionCount" json:"partition_count,omitempty"` |
|||
} |
|||
|
|||
func (m *PublishResponse_ConfigMessage) Reset() { *m = PublishResponse_ConfigMessage{} } |
|||
func (m *PublishResponse_ConfigMessage) String() string { return proto.CompactTextString(m) } |
|||
func (*PublishResponse_ConfigMessage) ProtoMessage() {} |
|||
func (*PublishResponse_ConfigMessage) Descriptor() ([]byte, []int) { |
|||
return fileDescriptor0, []int{4, 0} |
|||
} |
|||
|
|||
func (m *PublishResponse_ConfigMessage) GetPartitionCount() int32 { |
|||
if m != nil { |
|||
return m.PartitionCount |
|||
} |
|||
return 0 |
|||
} |
|||
|
|||
type PublishResponse_RedirectMessage struct { |
|||
NewBroker string `protobuf:"bytes,1,opt,name=new_broker,json=newBroker" json:"new_broker,omitempty"` |
|||
} |
|||
|
|||
func (m *PublishResponse_RedirectMessage) Reset() { *m = PublishResponse_RedirectMessage{} } |
|||
func (m *PublishResponse_RedirectMessage) String() string { return proto.CompactTextString(m) } |
|||
func (*PublishResponse_RedirectMessage) ProtoMessage() {} |
|||
func (*PublishResponse_RedirectMessage) Descriptor() ([]byte, []int) { |
|||
return fileDescriptor0, []int{4, 1} |
|||
} |
|||
|
|||
func (m *PublishResponse_RedirectMessage) GetNewBroker() string { |
|||
if m != nil { |
|||
return m.NewBroker |
|||
} |
|||
return "" |
|||
} |
|||
|
|||
type ConfigureTopicRequest struct { |
|||
Namespace string `protobuf:"bytes,1,opt,name=namespace" json:"namespace,omitempty"` |
|||
Topic string `protobuf:"bytes,2,opt,name=topic" json:"topic,omitempty"` |
|||
Configuration *TopicConfiguration `protobuf:"bytes,3,opt,name=configuration" json:"configuration,omitempty"` |
|||
} |
|||
|
|||
func (m *ConfigureTopicRequest) Reset() { *m = ConfigureTopicRequest{} } |
|||
func (m *ConfigureTopicRequest) String() string { return proto.CompactTextString(m) } |
|||
func (*ConfigureTopicRequest) ProtoMessage() {} |
|||
func (*ConfigureTopicRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{5} } |
|||
|
|||
func (m *ConfigureTopicRequest) GetNamespace() string { |
|||
if m != nil { |
|||
return m.Namespace |
|||
} |
|||
return "" |
|||
} |
|||
|
|||
func (m *ConfigureTopicRequest) GetTopic() string { |
|||
if m != nil { |
|||
return m.Topic |
|||
} |
|||
return "" |
|||
} |
|||
|
|||
func (m *ConfigureTopicRequest) GetConfiguration() *TopicConfiguration { |
|||
if m != nil { |
|||
return m.Configuration |
|||
} |
|||
return nil |
|||
} |
|||
|
|||
type ConfigureTopicResponse struct { |
|||
} |
|||
|
|||
func (m *ConfigureTopicResponse) Reset() { *m = ConfigureTopicResponse{} } |
|||
func (m *ConfigureTopicResponse) String() string { return proto.CompactTextString(m) } |
|||
func (*ConfigureTopicResponse) ProtoMessage() {} |
|||
func (*ConfigureTopicResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{6} } |
|||
|
|||
type GetTopicConfigurationRequest struct { |
|||
Namespace string `protobuf:"bytes,1,opt,name=namespace" json:"namespace,omitempty"` |
|||
Topic string `protobuf:"bytes,2,opt,name=topic" json:"topic,omitempty"` |
|||
} |
|||
|
|||
func (m *GetTopicConfigurationRequest) Reset() { *m = GetTopicConfigurationRequest{} } |
|||
func (m *GetTopicConfigurationRequest) String() string { return proto.CompactTextString(m) } |
|||
func (*GetTopicConfigurationRequest) ProtoMessage() {} |
|||
func (*GetTopicConfigurationRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{7} } |
|||
|
|||
func (m *GetTopicConfigurationRequest) GetNamespace() string { |
|||
if m != nil { |
|||
return m.Namespace |
|||
} |
|||
return "" |
|||
} |
|||
|
|||
func (m *GetTopicConfigurationRequest) GetTopic() string { |
|||
if m != nil { |
|||
return m.Topic |
|||
} |
|||
return "" |
|||
} |
|||
|
|||
type GetTopicConfigurationResponse struct { |
|||
Configuration *TopicConfiguration `protobuf:"bytes,1,opt,name=configuration" json:"configuration,omitempty"` |
|||
} |
|||
|
|||
func (m *GetTopicConfigurationResponse) Reset() { *m = GetTopicConfigurationResponse{} } |
|||
func (m *GetTopicConfigurationResponse) String() string { return proto.CompactTextString(m) } |
|||
func (*GetTopicConfigurationResponse) ProtoMessage() {} |
|||
func (*GetTopicConfigurationResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{8} } |
|||
|
|||
func (m *GetTopicConfigurationResponse) GetConfiguration() *TopicConfiguration { |
|||
if m != nil { |
|||
return m.Configuration |
|||
} |
|||
return nil |
|||
} |
|||
|
|||
type TopicConfiguration struct { |
|||
PartitionCount int32 `protobuf:"varint,1,opt,name=partition_count,json=partitionCount" json:"partition_count,omitempty"` |
|||
Collection string `protobuf:"bytes,2,opt,name=collection" json:"collection,omitempty"` |
|||
Replication string `protobuf:"bytes,3,opt,name=replication" json:"replication,omitempty"` |
|||
} |
|||
|
|||
func (m *TopicConfiguration) Reset() { *m = TopicConfiguration{} } |
|||
func (m *TopicConfiguration) String() string { return proto.CompactTextString(m) } |
|||
func (*TopicConfiguration) ProtoMessage() {} |
|||
func (*TopicConfiguration) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{9} } |
|||
|
|||
func (m *TopicConfiguration) GetPartitionCount() int32 { |
|||
if m != nil { |
|||
return m.PartitionCount |
|||
} |
|||
return 0 |
|||
} |
|||
|
|||
func (m *TopicConfiguration) GetCollection() string { |
|||
if m != nil { |
|||
return m.Collection |
|||
} |
|||
return "" |
|||
} |
|||
|
|||
func (m *TopicConfiguration) GetReplication() string { |
|||
if m != nil { |
|||
return m.Replication |
|||
} |
|||
return "" |
|||
} |
|||
|
|||
func init() { |
|||
proto.RegisterType((*SubscriberMessage)(nil), "messaging_pb.SubscriberMessage") |
|||
proto.RegisterType((*SubscriberMessage_InitMessage)(nil), "messaging_pb.SubscriberMessage.InitMessage") |
|||
proto.RegisterType((*SubscriberMessage_AckMessage)(nil), "messaging_pb.SubscriberMessage.AckMessage") |
|||
proto.RegisterType((*Message)(nil), "messaging_pb.Message") |
|||
proto.RegisterType((*BrokerMessage)(nil), "messaging_pb.BrokerMessage") |
|||
proto.RegisterType((*BrokerMessage_RedirectMessage)(nil), "messaging_pb.BrokerMessage.RedirectMessage") |
|||
proto.RegisterType((*PublishRequest)(nil), "messaging_pb.PublishRequest") |
|||
proto.RegisterType((*PublishRequest_InitMessage)(nil), "messaging_pb.PublishRequest.InitMessage") |
|||
proto.RegisterType((*PublishRequest_DataMessage)(nil), "messaging_pb.PublishRequest.DataMessage") |
|||
proto.RegisterType((*PublishResponse)(nil), "messaging_pb.PublishResponse") |
|||
proto.RegisterType((*PublishResponse_ConfigMessage)(nil), "messaging_pb.PublishResponse.ConfigMessage") |
|||
proto.RegisterType((*PublishResponse_RedirectMessage)(nil), "messaging_pb.PublishResponse.RedirectMessage") |
|||
proto.RegisterType((*ConfigureTopicRequest)(nil), "messaging_pb.ConfigureTopicRequest") |
|||
proto.RegisterType((*ConfigureTopicResponse)(nil), "messaging_pb.ConfigureTopicResponse") |
|||
proto.RegisterType((*GetTopicConfigurationRequest)(nil), "messaging_pb.GetTopicConfigurationRequest") |
|||
proto.RegisterType((*GetTopicConfigurationResponse)(nil), "messaging_pb.GetTopicConfigurationResponse") |
|||
proto.RegisterType((*TopicConfiguration)(nil), "messaging_pb.TopicConfiguration") |
|||
proto.RegisterEnum("messaging_pb.SubscriberMessage_InitMessage_StartPosition", SubscriberMessage_InitMessage_StartPosition_name, SubscriberMessage_InitMessage_StartPosition_value) |
|||
} |
|||
|
|||
// Reference imports to suppress errors if they are not otherwise used.
|
|||
var _ context.Context |
|||
var _ grpc.ClientConn |
|||
|
|||
// This is a compile-time assertion to ensure that this generated file
|
|||
// is compatible with the grpc package it is being compiled against.
|
|||
const _ = grpc.SupportPackageIsVersion4 |
|||
|
|||
// Client API for SeaweedMessaging service
|
|||
|
|||
type SeaweedMessagingClient interface { |
|||
Subscribe(ctx context.Context, opts ...grpc.CallOption) (SeaweedMessaging_SubscribeClient, error) |
|||
Publish(ctx context.Context, opts ...grpc.CallOption) (SeaweedMessaging_PublishClient, error) |
|||
ConfigureTopic(ctx context.Context, in *ConfigureTopicRequest, opts ...grpc.CallOption) (*ConfigureTopicResponse, error) |
|||
GetTopicConfiguration(ctx context.Context, in *GetTopicConfigurationRequest, opts ...grpc.CallOption) (*GetTopicConfigurationResponse, error) |
|||
} |
|||
|
|||
type seaweedMessagingClient struct { |
|||
cc *grpc.ClientConn |
|||
} |
|||
|
|||
func NewSeaweedMessagingClient(cc *grpc.ClientConn) SeaweedMessagingClient { |
|||
return &seaweedMessagingClient{cc} |
|||
} |
|||
|
|||
func (c *seaweedMessagingClient) Subscribe(ctx context.Context, opts ...grpc.CallOption) (SeaweedMessaging_SubscribeClient, error) { |
|||
stream, err := grpc.NewClientStream(ctx, &_SeaweedMessaging_serviceDesc.Streams[0], c.cc, "/messaging_pb.SeaweedMessaging/Subscribe", opts...) |
|||
if err != nil { |
|||
return nil, err |
|||
} |
|||
x := &seaweedMessagingSubscribeClient{stream} |
|||
return x, nil |
|||
} |
|||
|
|||
type SeaweedMessaging_SubscribeClient interface { |
|||
Send(*SubscriberMessage) error |
|||
Recv() (*BrokerMessage, error) |
|||
grpc.ClientStream |
|||
} |
|||
|
|||
type seaweedMessagingSubscribeClient struct { |
|||
grpc.ClientStream |
|||
} |
|||
|
|||
func (x *seaweedMessagingSubscribeClient) Send(m *SubscriberMessage) error { |
|||
return x.ClientStream.SendMsg(m) |
|||
} |
|||
|
|||
func (x *seaweedMessagingSubscribeClient) Recv() (*BrokerMessage, error) { |
|||
m := new(BrokerMessage) |
|||
if err := x.ClientStream.RecvMsg(m); err != nil { |
|||
return nil, err |
|||
} |
|||
return m, nil |
|||
} |
|||
|
|||
func (c *seaweedMessagingClient) Publish(ctx context.Context, opts ...grpc.CallOption) (SeaweedMessaging_PublishClient, error) { |
|||
stream, err := grpc.NewClientStream(ctx, &_SeaweedMessaging_serviceDesc.Streams[1], c.cc, "/messaging_pb.SeaweedMessaging/Publish", opts...) |
|||
if err != nil { |
|||
return nil, err |
|||
} |
|||
x := &seaweedMessagingPublishClient{stream} |
|||
return x, nil |
|||
} |
|||
|
|||
type SeaweedMessaging_PublishClient interface { |
|||
Send(*PublishRequest) error |
|||
Recv() (*PublishResponse, error) |
|||
grpc.ClientStream |
|||
} |
|||
|
|||
type seaweedMessagingPublishClient struct { |
|||
grpc.ClientStream |
|||
} |
|||
|
|||
func (x *seaweedMessagingPublishClient) Send(m *PublishRequest) error { |
|||
return x.ClientStream.SendMsg(m) |
|||
} |
|||
|
|||
func (x *seaweedMessagingPublishClient) Recv() (*PublishResponse, error) { |
|||
m := new(PublishResponse) |
|||
if err := x.ClientStream.RecvMsg(m); err != nil { |
|||
return nil, err |
|||
} |
|||
return m, nil |
|||
} |
|||
|
|||
func (c *seaweedMessagingClient) ConfigureTopic(ctx context.Context, in *ConfigureTopicRequest, opts ...grpc.CallOption) (*ConfigureTopicResponse, error) { |
|||
out := new(ConfigureTopicResponse) |
|||
err := grpc.Invoke(ctx, "/messaging_pb.SeaweedMessaging/ConfigureTopic", in, out, c.cc, opts...) |
|||
if err != nil { |
|||
return nil, err |
|||
} |
|||
return out, nil |
|||
} |
|||
|
|||
func (c *seaweedMessagingClient) GetTopicConfiguration(ctx context.Context, in *GetTopicConfigurationRequest, opts ...grpc.CallOption) (*GetTopicConfigurationResponse, error) { |
|||
out := new(GetTopicConfigurationResponse) |
|||
err := grpc.Invoke(ctx, "/messaging_pb.SeaweedMessaging/GetTopicConfiguration", in, out, c.cc, opts...) |
|||
if err != nil { |
|||
return nil, err |
|||
} |
|||
return out, nil |
|||
} |
|||
|
|||
// Server API for SeaweedMessaging service
|
|||
|
|||
type SeaweedMessagingServer interface { |
|||
Subscribe(SeaweedMessaging_SubscribeServer) error |
|||
Publish(SeaweedMessaging_PublishServer) error |
|||
ConfigureTopic(context.Context, *ConfigureTopicRequest) (*ConfigureTopicResponse, error) |
|||
GetTopicConfiguration(context.Context, *GetTopicConfigurationRequest) (*GetTopicConfigurationResponse, error) |
|||
} |
|||
|
|||
func RegisterSeaweedMessagingServer(s *grpc.Server, srv SeaweedMessagingServer) { |
|||
s.RegisterService(&_SeaweedMessaging_serviceDesc, srv) |
|||
} |
|||
|
|||
func _SeaweedMessaging_Subscribe_Handler(srv interface{}, stream grpc.ServerStream) error { |
|||
return srv.(SeaweedMessagingServer).Subscribe(&seaweedMessagingSubscribeServer{stream}) |
|||
} |
|||
|
|||
type SeaweedMessaging_SubscribeServer interface { |
|||
Send(*BrokerMessage) error |
|||
Recv() (*SubscriberMessage, error) |
|||
grpc.ServerStream |
|||
} |
|||
|
|||
type seaweedMessagingSubscribeServer struct { |
|||
grpc.ServerStream |
|||
} |
|||
|
|||
func (x *seaweedMessagingSubscribeServer) Send(m *BrokerMessage) error { |
|||
return x.ServerStream.SendMsg(m) |
|||
} |
|||
|
|||
func (x *seaweedMessagingSubscribeServer) Recv() (*SubscriberMessage, error) { |
|||
m := new(SubscriberMessage) |
|||
if err := x.ServerStream.RecvMsg(m); err != nil { |
|||
return nil, err |
|||
} |
|||
return m, nil |
|||
} |
|||
|
|||
func _SeaweedMessaging_Publish_Handler(srv interface{}, stream grpc.ServerStream) error { |
|||
return srv.(SeaweedMessagingServer).Publish(&seaweedMessagingPublishServer{stream}) |
|||
} |
|||
|
|||
type SeaweedMessaging_PublishServer interface { |
|||
Send(*PublishResponse) error |
|||
Recv() (*PublishRequest, error) |
|||
grpc.ServerStream |
|||
} |
|||
|
|||
type seaweedMessagingPublishServer struct { |
|||
grpc.ServerStream |
|||
} |
|||
|
|||
func (x *seaweedMessagingPublishServer) Send(m *PublishResponse) error { |
|||
return x.ServerStream.SendMsg(m) |
|||
} |
|||
|
|||
func (x *seaweedMessagingPublishServer) Recv() (*PublishRequest, error) { |
|||
m := new(PublishRequest) |
|||
if err := x.ServerStream.RecvMsg(m); err != nil { |
|||
return nil, err |
|||
} |
|||
return m, nil |
|||
} |
|||
|
|||
func _SeaweedMessaging_ConfigureTopic_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { |
|||
in := new(ConfigureTopicRequest) |
|||
if err := dec(in); err != nil { |
|||
return nil, err |
|||
} |
|||
if interceptor == nil { |
|||
return srv.(SeaweedMessagingServer).ConfigureTopic(ctx, in) |
|||
} |
|||
info := &grpc.UnaryServerInfo{ |
|||
Server: srv, |
|||
FullMethod: "/messaging_pb.SeaweedMessaging/ConfigureTopic", |
|||
} |
|||
handler := func(ctx context.Context, req interface{}) (interface{}, error) { |
|||
return srv.(SeaweedMessagingServer).ConfigureTopic(ctx, req.(*ConfigureTopicRequest)) |
|||
} |
|||
return interceptor(ctx, in, info, handler) |
|||
} |
|||
|
|||
func _SeaweedMessaging_GetTopicConfiguration_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { |
|||
in := new(GetTopicConfigurationRequest) |
|||
if err := dec(in); err != nil { |
|||
return nil, err |
|||
} |
|||
if interceptor == nil { |
|||
return srv.(SeaweedMessagingServer).GetTopicConfiguration(ctx, in) |
|||
} |
|||
info := &grpc.UnaryServerInfo{ |
|||
Server: srv, |
|||
FullMethod: "/messaging_pb.SeaweedMessaging/GetTopicConfiguration", |
|||
} |
|||
handler := func(ctx context.Context, req interface{}) (interface{}, error) { |
|||
return srv.(SeaweedMessagingServer).GetTopicConfiguration(ctx, req.(*GetTopicConfigurationRequest)) |
|||
} |
|||
return interceptor(ctx, in, info, handler) |
|||
} |
|||
|
|||
var _SeaweedMessaging_serviceDesc = grpc.ServiceDesc{ |
|||
ServiceName: "messaging_pb.SeaweedMessaging", |
|||
HandlerType: (*SeaweedMessagingServer)(nil), |
|||
Methods: []grpc.MethodDesc{ |
|||
{ |
|||
MethodName: "ConfigureTopic", |
|||
Handler: _SeaweedMessaging_ConfigureTopic_Handler, |
|||
}, |
|||
{ |
|||
MethodName: "GetTopicConfiguration", |
|||
Handler: _SeaweedMessaging_GetTopicConfiguration_Handler, |
|||
}, |
|||
}, |
|||
Streams: []grpc.StreamDesc{ |
|||
{ |
|||
StreamName: "Subscribe", |
|||
Handler: _SeaweedMessaging_Subscribe_Handler, |
|||
ServerStreams: true, |
|||
ClientStreams: true, |
|||
}, |
|||
{ |
|||
StreamName: "Publish", |
|||
Handler: _SeaweedMessaging_Publish_Handler, |
|||
ServerStreams: true, |
|||
ClientStreams: true, |
|||
}, |
|||
}, |
|||
Metadata: "messaging.proto", |
|||
} |
|||
|
|||
func init() { proto.RegisterFile("messaging.proto", fileDescriptor0) } |
|||
|
|||
var fileDescriptor0 = []byte{ |
|||
// 829 bytes of a gzipped FileDescriptorProto
|
|||
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xb4, 0x56, 0x4d, 0x8f, 0xe3, 0x44, |
|||
0x10, 0xdd, 0xb6, 0x93, 0xcc, 0xba, 0xf2, 0x49, 0x8b, 0x41, 0x91, 0x99, 0x01, 0xcb, 0x8b, 0x44, |
|||
0x60, 0x84, 0x35, 0x0a, 0x42, 0x1a, 0x56, 0x2b, 0xa1, 0x24, 0x84, 0x25, 0xd2, 0x84, 0x8d, 0x3a, |
|||
0xb9, 0xa2, 0xc8, 0x71, 0x7a, 0xb3, 0x56, 0x12, 0xdb, 0xb8, 0x3b, 0x8c, 0xe6, 0xc4, 0x01, 0xae, |
|||
0x9c, 0xf8, 0x27, 0x48, 0xfc, 0x00, 0xb8, 0xf3, 0x9f, 0x90, 0xdb, 0xdf, 0x49, 0x26, 0x13, 0x22, |
|||
0xcd, 0xcd, 0x2e, 0xbf, 0x7a, 0xaf, 0xba, 0xea, 0x95, 0x6d, 0xa8, 0xaf, 0x29, 0x63, 0xe6, 0xc2, |
|||
0x76, 0x16, 0x86, 0xe7, 0xbb, 0xdc, 0xc5, 0x95, 0x24, 0x30, 0xf5, 0x66, 0xfa, 0xaf, 0x05, 0x78, |
|||
0x6f, 0xbc, 0x99, 0x31, 0xcb, 0xb7, 0x67, 0xd4, 0x1f, 0x8a, 0x47, 0x14, 0x7f, 0x03, 0x05, 0xdb, |
|||
0xb1, 0x79, 0x13, 0x69, 0xa8, 0x55, 0x6e, 0x5f, 0x19, 0xd9, 0x14, 0x63, 0x07, 0x6e, 0x0c, 0x1c, |
|||
0x9b, 0x47, 0xd7, 0x44, 0x24, 0xe2, 0x57, 0x20, 0x9b, 0xd6, 0xb2, 0x29, 0x89, 0xfc, 0xcf, 0x1f, |
|||
0xcb, 0xef, 0x58, 0xcb, 0x38, 0x3d, 0x48, 0x53, 0xff, 0x96, 0xa0, 0x9c, 0xe1, 0xc4, 0x17, 0xa0, |
|||
0x38, 0xe6, 0x9a, 0x32, 0xcf, 0xb4, 0xa8, 0xa8, 0x49, 0x21, 0x69, 0x00, 0xbf, 0x0f, 0x45, 0xee, |
|||
0x7a, 0xb6, 0x25, 0xd4, 0x14, 0x12, 0xde, 0x04, 0x39, 0x9e, 0xe9, 0x73, 0x9b, 0xdb, 0xae, 0xd3, |
|||
0x94, 0x35, 0xd4, 0x2a, 0x92, 0x34, 0x80, 0xa7, 0x50, 0x65, 0xdc, 0xf4, 0xf9, 0xc8, 0x65, 0x21, |
|||
0xa2, 0xa0, 0xa1, 0x56, 0xad, 0xfd, 0xf5, 0xff, 0x38, 0xa9, 0x31, 0xce, 0x12, 0x90, 0x3c, 0x1f, |
|||
0xd6, 0xa0, 0xcc, 0xed, 0x35, 0x65, 0xdc, 0x5c, 0x7b, 0x3f, 0xb0, 0x66, 0x51, 0x43, 0x2d, 0x99, |
|||
0x64, 0x43, 0xf8, 0x05, 0x54, 0x59, 0xc2, 0x3f, 0xb5, 0xe7, 0xcd, 0x92, 0x28, 0xbf, 0x92, 0x06, |
|||
0x07, 0x73, 0xfd, 0x06, 0xaa, 0x39, 0x19, 0x0c, 0x50, 0xba, 0xed, 0x4c, 0xfa, 0xe3, 0x49, 0xe3, |
|||
0x19, 0xae, 0xc0, 0xf3, 0x7e, 0x87, 0xdc, 0x0e, 0x82, 0x3b, 0x84, 0xab, 0xa0, 0x4c, 0x06, 0xc3, |
|||
0xfe, 0x78, 0xd2, 0x19, 0x8e, 0x1a, 0x92, 0x7a, 0x05, 0x90, 0xb6, 0x15, 0x5f, 0x02, 0x84, 0x27, |
|||
0xa3, 0x81, 0x12, 0x12, 0xd5, 0x28, 0x51, 0x64, 0x30, 0xd7, 0xff, 0x45, 0x70, 0x16, 0x43, 0x35, |
|||
0x50, 0x92, 0x32, 0x43, 0x64, 0x57, 0xba, 0x46, 0x24, 0x0d, 0xe2, 0x06, 0xc8, 0x4b, 0x7a, 0x2f, |
|||
0xda, 0x5d, 0x21, 0xc1, 0x65, 0x30, 0x82, 0x9f, 0xcd, 0xd5, 0x86, 0x8a, 0x46, 0x57, 0x48, 0x78, |
|||
0x83, 0x5f, 0xc1, 0xd9, 0x3b, 0x6a, 0xce, 0xa9, 0xcf, 0x9a, 0x05, 0x4d, 0x6e, 0x95, 0xdb, 0x7a, |
|||
0xbe, 0xbd, 0x71, 0x23, 0xbf, 0x0f, 0x41, 0x7d, 0x87, 0xfb, 0xf7, 0x24, 0x4e, 0x51, 0x5f, 0x42, |
|||
0x25, 0xfb, 0x20, 0x56, 0x0d, 0xc7, 0x9f, 0x57, 0x95, 0x32, 0xaa, 0x2f, 0xa5, 0x1b, 0xa4, 0xff, |
|||
0x85, 0xa0, 0xda, 0xf5, 0xdd, 0x65, 0xea, 0xe8, 0xcf, 0xa0, 0x30, 0x37, 0xb9, 0x19, 0x39, 0xfa, |
|||
0x7c, 0x6f, 0x21, 0x44, 0x40, 0xf0, 0x6b, 0x78, 0xee, 0xd3, 0xb9, 0xed, 0x53, 0x8b, 0x47, 0x06, |
|||
0xde, 0x5a, 0x80, 0x1c, 0xb3, 0x41, 0x22, 0x6c, 0x4c, 0x92, 0x24, 0xab, 0xd7, 0x50, 0xdf, 0x7a, |
|||
0x18, 0xcc, 0xc1, 0xa1, 0x77, 0xd3, 0x99, 0x60, 0x48, 0xac, 0x4c, 0xef, 0x42, 0x4a, 0xfd, 0x4f, |
|||
0x19, 0x6a, 0xa3, 0xcd, 0x6c, 0x65, 0xb3, 0x77, 0x84, 0xfe, 0xb4, 0xa1, 0x2c, 0xd8, 0xa4, 0xec, |
|||
0x2a, 0xb6, 0xf2, 0x95, 0xe4, 0xb1, 0x7b, 0xf7, 0x30, 0x3c, 0xb6, 0x74, 0x44, 0xf6, 0xb7, 0x26, |
|||
0x37, 0x73, 0x9d, 0x50, 0xa7, 0x4f, 0xbc, 0x86, 0xea, 0x3f, 0x08, 0xca, 0x19, 0xd9, 0xec, 0x8c, |
|||
0x2b, 0x07, 0x66, 0x8c, 0xdf, 0xa4, 0xce, 0x92, 0x85, 0xb3, 0xbe, 0x3a, 0xf6, 0x64, 0x4f, 0x60, |
|||
0xb6, 0xdf, 0x25, 0xa8, 0x27, 0x82, 0xcc, 0x73, 0x1d, 0x46, 0x71, 0x0f, 0x4a, 0x96, 0xeb, 0xbc, |
|||
0xb5, 0x17, 0xfb, 0x5f, 0xa1, 0x5b, 0x70, 0xa3, 0x27, 0xb0, 0x71, 0xf3, 0xa3, 0x54, 0x3c, 0xd8, |
|||
0x31, 0xe2, 0x17, 0x87, 0x69, 0x1e, 0xb6, 0xe2, 0x0d, 0x54, 0x73, 0x1a, 0xf8, 0x53, 0xa8, 0x27, |
|||
0x63, 0x98, 0x5a, 0xee, 0xc6, 0x09, 0x1d, 0x56, 0x24, 0xb5, 0x24, 0xdc, 0x0b, 0xa2, 0x27, 0x98, |
|||
0xf8, 0x0f, 0x04, 0xe7, 0xa1, 0xd8, 0xc6, 0xa7, 0x93, 0xc0, 0x05, 0xb1, 0x97, 0x4f, 0x31, 0xd0, |
|||
0x77, 0x50, 0xb5, 0x22, 0x32, 0x33, 0x31, 0x51, 0xb9, 0xad, 0xe5, 0x3b, 0x21, 0x64, 0x7a, 0x59, |
|||
0x1c, 0xc9, 0xa7, 0xe9, 0x4d, 0xf8, 0x60, 0xbb, 0xa8, 0xb0, 0x6b, 0x3a, 0x81, 0x8b, 0xd7, 0x94, |
|||
0xef, 0x61, 0x38, 0xbd, 0x6a, 0x7d, 0x01, 0x97, 0x0f, 0x70, 0x46, 0x06, 0xd9, 0x39, 0x16, 0x3a, |
|||
0xed, 0x58, 0xbf, 0x00, 0xde, 0x05, 0x1d, 0x3d, 0x5d, 0xfc, 0x11, 0x80, 0xe5, 0xae, 0x56, 0xd4, |
|||
0x12, 0x35, 0x84, 0x47, 0xc8, 0x44, 0x82, 0xcf, 0x98, 0x4f, 0xbd, 0x95, 0x6d, 0xa5, 0xbd, 0x57, |
|||
0x48, 0x36, 0xd4, 0xfe, 0x4d, 0x86, 0xc6, 0x98, 0x9a, 0x77, 0x94, 0xce, 0x87, 0x71, 0xe9, 0xf8, |
|||
0x0d, 0x28, 0xc9, 0xb7, 0x13, 0x7f, 0xfc, 0xc8, 0x47, 0x55, 0xfd, 0xf0, 0xc0, 0xeb, 0x55, 0x7f, |
|||
0xd6, 0x42, 0xd7, 0x08, 0xdf, 0xc2, 0x59, 0x64, 0x76, 0x7c, 0x71, 0x68, 0xd5, 0xd5, 0xcb, 0x83, |
|||
0x1b, 0x12, 0xb1, 0xfd, 0x08, 0xb5, 0xbc, 0x17, 0xf0, 0x8b, 0x7c, 0xda, 0x5e, 0xfb, 0xaa, 0x9f, |
|||
0x1c, 0x06, 0xc5, 0x12, 0xd8, 0x87, 0xf3, 0xbd, 0xc3, 0xc7, 0x5b, 0x3f, 0x42, 0x87, 0x5c, 0xa7, |
|||
0x5e, 0x1d, 0x85, 0x8d, 0x35, 0xbb, 0x3a, 0x34, 0x58, 0x38, 0x85, 0xb7, 0xcc, 0xb0, 0x56, 0x36, |
|||
0x75, 0x78, 0xb7, 0x96, 0x0c, 0x64, 0x14, 0xfc, 0xf9, 0xcd, 0x4a, 0xe2, 0x07, 0xf0, 0xcb, 0xff, |
|||
0x02, 0x00, 0x00, 0xff, 0xff, 0xed, 0x8d, 0x77, 0xac, 0x13, 0x0a, 0x00, 0x00, |
|||
} |
|||
@ -1,66 +0,0 @@ |
|||
syntax = "proto3"; |
|||
|
|||
package queue_pb; |
|||
|
|||
option java_package = "seaweedfs.client"; |
|||
option java_outer_classname = "QueueProto"; |
|||
|
|||
////////////////////////////////////////////////// |
|||
|
|||
service SeaweedQueue { |
|||
|
|||
rpc StreamWrite (stream WriteMessageRequest) returns (stream WriteMessageResponse) { |
|||
} |
|||
|
|||
rpc StreamRead (ReadMessageRequest) returns (stream ReadMessageResponse) { |
|||
} |
|||
|
|||
rpc ConfigureTopic (ConfigureTopicRequest) returns (ConfigureTopicResponse) { |
|||
} |
|||
|
|||
rpc DeleteTopic (DeleteTopicRequest) returns (DeleteTopicResponse) { |
|||
} |
|||
|
|||
} |
|||
|
|||
////////////////////////////////////////////////// |
|||
|
|||
|
|||
message WriteMessageRequest { |
|||
string topic = 1; |
|||
int64 event_ns = 2; |
|||
bytes partition_key = 3; |
|||
bytes data = 4; |
|||
} |
|||
|
|||
message WriteMessageResponse { |
|||
string error = 1; |
|||
int64 ack_ns = 2; |
|||
} |
|||
|
|||
message ReadMessageRequest { |
|||
string topic = 1; |
|||
int64 start_ns = 2; |
|||
} |
|||
|
|||
message ReadMessageResponse { |
|||
string error = 1; |
|||
int64 event_ns = 2; |
|||
bytes data = 3; |
|||
} |
|||
|
|||
message ConfigureTopicRequest { |
|||
string topic = 1; |
|||
int64 ttl_seconds = 2; |
|||
int32 partition_count = 3; |
|||
} |
|||
message ConfigureTopicResponse { |
|||
string error = 1; |
|||
} |
|||
|
|||
message DeleteTopicRequest { |
|||
string topic = 1; |
|||
} |
|||
message DeleteTopicResponse { |
|||
string error = 1; |
|||
} |
|||
@ -1,516 +0,0 @@ |
|||
// Code generated by protoc-gen-go.
|
|||
// source: queue.proto
|
|||
// DO NOT EDIT!
|
|||
|
|||
/* |
|||
Package queue_pb is a generated protocol buffer package. |
|||
|
|||
It is generated from these files: |
|||
queue.proto |
|||
|
|||
It has these top-level messages: |
|||
WriteMessageRequest |
|||
WriteMessageResponse |
|||
ReadMessageRequest |
|||
ReadMessageResponse |
|||
ConfigureTopicRequest |
|||
ConfigureTopicResponse |
|||
DeleteTopicRequest |
|||
DeleteTopicResponse |
|||
*/ |
|||
package queue_pb |
|||
|
|||
import proto "github.com/golang/protobuf/proto" |
|||
import fmt "fmt" |
|||
import math "math" |
|||
|
|||
import ( |
|||
context "golang.org/x/net/context" |
|||
grpc "google.golang.org/grpc" |
|||
) |
|||
|
|||
// Reference imports to suppress errors if they are not otherwise used.
|
|||
var _ = proto.Marshal |
|||
var _ = fmt.Errorf |
|||
var _ = math.Inf |
|||
|
|||
// This is a compile-time assertion to ensure that this generated file
|
|||
// is compatible with the proto package it is being compiled against.
|
|||
// A compilation error at this line likely means your copy of the
|
|||
// proto package needs to be updated.
|
|||
const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package
|
|||
|
|||
type WriteMessageRequest struct { |
|||
Topic string `protobuf:"bytes,1,opt,name=topic" json:"topic,omitempty"` |
|||
EventNs int64 `protobuf:"varint,2,opt,name=event_ns,json=eventNs" json:"event_ns,omitempty"` |
|||
PartitionKey []byte `protobuf:"bytes,3,opt,name=partition_key,json=partitionKey,proto3" json:"partition_key,omitempty"` |
|||
Data []byte `protobuf:"bytes,4,opt,name=data,proto3" json:"data,omitempty"` |
|||
} |
|||
|
|||
func (m *WriteMessageRequest) Reset() { *m = WriteMessageRequest{} } |
|||
func (m *WriteMessageRequest) String() string { return proto.CompactTextString(m) } |
|||
func (*WriteMessageRequest) ProtoMessage() {} |
|||
func (*WriteMessageRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{0} } |
|||
|
|||
func (m *WriteMessageRequest) GetTopic() string { |
|||
if m != nil { |
|||
return m.Topic |
|||
} |
|||
return "" |
|||
} |
|||
|
|||
func (m *WriteMessageRequest) GetEventNs() int64 { |
|||
if m != nil { |
|||
return m.EventNs |
|||
} |
|||
return 0 |
|||
} |
|||
|
|||
func (m *WriteMessageRequest) GetPartitionKey() []byte { |
|||
if m != nil { |
|||
return m.PartitionKey |
|||
} |
|||
return nil |
|||
} |
|||
|
|||
func (m *WriteMessageRequest) GetData() []byte { |
|||
if m != nil { |
|||
return m.Data |
|||
} |
|||
return nil |
|||
} |
|||
|
|||
type WriteMessageResponse struct { |
|||
Error string `protobuf:"bytes,1,opt,name=error" json:"error,omitempty"` |
|||
AckNs int64 `protobuf:"varint,2,opt,name=ack_ns,json=ackNs" json:"ack_ns,omitempty"` |
|||
} |
|||
|
|||
func (m *WriteMessageResponse) Reset() { *m = WriteMessageResponse{} } |
|||
func (m *WriteMessageResponse) String() string { return proto.CompactTextString(m) } |
|||
func (*WriteMessageResponse) ProtoMessage() {} |
|||
func (*WriteMessageResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{1} } |
|||
|
|||
func (m *WriteMessageResponse) GetError() string { |
|||
if m != nil { |
|||
return m.Error |
|||
} |
|||
return "" |
|||
} |
|||
|
|||
func (m *WriteMessageResponse) GetAckNs() int64 { |
|||
if m != nil { |
|||
return m.AckNs |
|||
} |
|||
return 0 |
|||
} |
|||
|
|||
type ReadMessageRequest struct { |
|||
Topic string `protobuf:"bytes,1,opt,name=topic" json:"topic,omitempty"` |
|||
StartNs int64 `protobuf:"varint,2,opt,name=start_ns,json=startNs" json:"start_ns,omitempty"` |
|||
} |
|||
|
|||
func (m *ReadMessageRequest) Reset() { *m = ReadMessageRequest{} } |
|||
func (m *ReadMessageRequest) String() string { return proto.CompactTextString(m) } |
|||
func (*ReadMessageRequest) ProtoMessage() {} |
|||
func (*ReadMessageRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{2} } |
|||
|
|||
func (m *ReadMessageRequest) GetTopic() string { |
|||
if m != nil { |
|||
return m.Topic |
|||
} |
|||
return "" |
|||
} |
|||
|
|||
func (m *ReadMessageRequest) GetStartNs() int64 { |
|||
if m != nil { |
|||
return m.StartNs |
|||
} |
|||
return 0 |
|||
} |
|||
|
|||
type ReadMessageResponse struct { |
|||
Error string `protobuf:"bytes,1,opt,name=error" json:"error,omitempty"` |
|||
EventNs int64 `protobuf:"varint,2,opt,name=event_ns,json=eventNs" json:"event_ns,omitempty"` |
|||
Data []byte `protobuf:"bytes,3,opt,name=data,proto3" json:"data,omitempty"` |
|||
} |
|||
|
|||
func (m *ReadMessageResponse) Reset() { *m = ReadMessageResponse{} } |
|||
func (m *ReadMessageResponse) String() string { return proto.CompactTextString(m) } |
|||
func (*ReadMessageResponse) ProtoMessage() {} |
|||
func (*ReadMessageResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{3} } |
|||
|
|||
func (m *ReadMessageResponse) GetError() string { |
|||
if m != nil { |
|||
return m.Error |
|||
} |
|||
return "" |
|||
} |
|||
|
|||
func (m *ReadMessageResponse) GetEventNs() int64 { |
|||
if m != nil { |
|||
return m.EventNs |
|||
} |
|||
return 0 |
|||
} |
|||
|
|||
func (m *ReadMessageResponse) GetData() []byte { |
|||
if m != nil { |
|||
return m.Data |
|||
} |
|||
return nil |
|||
} |
|||
|
|||
type ConfigureTopicRequest struct { |
|||
Topic string `protobuf:"bytes,1,opt,name=topic" json:"topic,omitempty"` |
|||
TtlSeconds int64 `protobuf:"varint,2,opt,name=ttl_seconds,json=ttlSeconds" json:"ttl_seconds,omitempty"` |
|||
PartitionCount int32 `protobuf:"varint,3,opt,name=partition_count,json=partitionCount" json:"partition_count,omitempty"` |
|||
} |
|||
|
|||
func (m *ConfigureTopicRequest) Reset() { *m = ConfigureTopicRequest{} } |
|||
func (m *ConfigureTopicRequest) String() string { return proto.CompactTextString(m) } |
|||
func (*ConfigureTopicRequest) ProtoMessage() {} |
|||
func (*ConfigureTopicRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{4} } |
|||
|
|||
func (m *ConfigureTopicRequest) GetTopic() string { |
|||
if m != nil { |
|||
return m.Topic |
|||
} |
|||
return "" |
|||
} |
|||
|
|||
func (m *ConfigureTopicRequest) GetTtlSeconds() int64 { |
|||
if m != nil { |
|||
return m.TtlSeconds |
|||
} |
|||
return 0 |
|||
} |
|||
|
|||
func (m *ConfigureTopicRequest) GetPartitionCount() int32 { |
|||
if m != nil { |
|||
return m.PartitionCount |
|||
} |
|||
return 0 |
|||
} |
|||
|
|||
type ConfigureTopicResponse struct { |
|||
Error string `protobuf:"bytes,1,opt,name=error" json:"error,omitempty"` |
|||
} |
|||
|
|||
func (m *ConfigureTopicResponse) Reset() { *m = ConfigureTopicResponse{} } |
|||
func (m *ConfigureTopicResponse) String() string { return proto.CompactTextString(m) } |
|||
func (*ConfigureTopicResponse) ProtoMessage() {} |
|||
func (*ConfigureTopicResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{5} } |
|||
|
|||
func (m *ConfigureTopicResponse) GetError() string { |
|||
if m != nil { |
|||
return m.Error |
|||
} |
|||
return "" |
|||
} |
|||
|
|||
type DeleteTopicRequest struct { |
|||
Topic string `protobuf:"bytes,1,opt,name=topic" json:"topic,omitempty"` |
|||
} |
|||
|
|||
func (m *DeleteTopicRequest) Reset() { *m = DeleteTopicRequest{} } |
|||
func (m *DeleteTopicRequest) String() string { return proto.CompactTextString(m) } |
|||
func (*DeleteTopicRequest) ProtoMessage() {} |
|||
func (*DeleteTopicRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{6} } |
|||
|
|||
func (m *DeleteTopicRequest) GetTopic() string { |
|||
if m != nil { |
|||
return m.Topic |
|||
} |
|||
return "" |
|||
} |
|||
|
|||
type DeleteTopicResponse struct { |
|||
Error string `protobuf:"bytes,1,opt,name=error" json:"error,omitempty"` |
|||
} |
|||
|
|||
func (m *DeleteTopicResponse) Reset() { *m = DeleteTopicResponse{} } |
|||
func (m *DeleteTopicResponse) String() string { return proto.CompactTextString(m) } |
|||
func (*DeleteTopicResponse) ProtoMessage() {} |
|||
func (*DeleteTopicResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{7} } |
|||
|
|||
func (m *DeleteTopicResponse) GetError() string { |
|||
if m != nil { |
|||
return m.Error |
|||
} |
|||
return "" |
|||
} |
|||
|
|||
func init() { |
|||
proto.RegisterType((*WriteMessageRequest)(nil), "queue_pb.WriteMessageRequest") |
|||
proto.RegisterType((*WriteMessageResponse)(nil), "queue_pb.WriteMessageResponse") |
|||
proto.RegisterType((*ReadMessageRequest)(nil), "queue_pb.ReadMessageRequest") |
|||
proto.RegisterType((*ReadMessageResponse)(nil), "queue_pb.ReadMessageResponse") |
|||
proto.RegisterType((*ConfigureTopicRequest)(nil), "queue_pb.ConfigureTopicRequest") |
|||
proto.RegisterType((*ConfigureTopicResponse)(nil), "queue_pb.ConfigureTopicResponse") |
|||
proto.RegisterType((*DeleteTopicRequest)(nil), "queue_pb.DeleteTopicRequest") |
|||
proto.RegisterType((*DeleteTopicResponse)(nil), "queue_pb.DeleteTopicResponse") |
|||
} |
|||
|
|||
// Reference imports to suppress errors if they are not otherwise used.
|
|||
var _ context.Context |
|||
var _ grpc.ClientConn |
|||
|
|||
// This is a compile-time assertion to ensure that this generated file
|
|||
// is compatible with the grpc package it is being compiled against.
|
|||
const _ = grpc.SupportPackageIsVersion4 |
|||
|
|||
// Client API for SeaweedQueue service
|
|||
|
|||
type SeaweedQueueClient interface { |
|||
StreamWrite(ctx context.Context, opts ...grpc.CallOption) (SeaweedQueue_StreamWriteClient, error) |
|||
StreamRead(ctx context.Context, in *ReadMessageRequest, opts ...grpc.CallOption) (SeaweedQueue_StreamReadClient, error) |
|||
ConfigureTopic(ctx context.Context, in *ConfigureTopicRequest, opts ...grpc.CallOption) (*ConfigureTopicResponse, error) |
|||
DeleteTopic(ctx context.Context, in *DeleteTopicRequest, opts ...grpc.CallOption) (*DeleteTopicResponse, error) |
|||
} |
|||
|
|||
type seaweedQueueClient struct { |
|||
cc *grpc.ClientConn |
|||
} |
|||
|
|||
func NewSeaweedQueueClient(cc *grpc.ClientConn) SeaweedQueueClient { |
|||
return &seaweedQueueClient{cc} |
|||
} |
|||
|
|||
func (c *seaweedQueueClient) StreamWrite(ctx context.Context, opts ...grpc.CallOption) (SeaweedQueue_StreamWriteClient, error) { |
|||
stream, err := grpc.NewClientStream(ctx, &_SeaweedQueue_serviceDesc.Streams[0], c.cc, "/queue_pb.SeaweedQueue/StreamWrite", opts...) |
|||
if err != nil { |
|||
return nil, err |
|||
} |
|||
x := &seaweedQueueStreamWriteClient{stream} |
|||
return x, nil |
|||
} |
|||
|
|||
type SeaweedQueue_StreamWriteClient interface { |
|||
Send(*WriteMessageRequest) error |
|||
Recv() (*WriteMessageResponse, error) |
|||
grpc.ClientStream |
|||
} |
|||
|
|||
type seaweedQueueStreamWriteClient struct { |
|||
grpc.ClientStream |
|||
} |
|||
|
|||
func (x *seaweedQueueStreamWriteClient) Send(m *WriteMessageRequest) error { |
|||
return x.ClientStream.SendMsg(m) |
|||
} |
|||
|
|||
func (x *seaweedQueueStreamWriteClient) Recv() (*WriteMessageResponse, error) { |
|||
m := new(WriteMessageResponse) |
|||
if err := x.ClientStream.RecvMsg(m); err != nil { |
|||
return nil, err |
|||
} |
|||
return m, nil |
|||
} |
|||
|
|||
func (c *seaweedQueueClient) StreamRead(ctx context.Context, in *ReadMessageRequest, opts ...grpc.CallOption) (SeaweedQueue_StreamReadClient, error) { |
|||
stream, err := grpc.NewClientStream(ctx, &_SeaweedQueue_serviceDesc.Streams[1], c.cc, "/queue_pb.SeaweedQueue/StreamRead", opts...) |
|||
if err != nil { |
|||
return nil, err |
|||
} |
|||
x := &seaweedQueueStreamReadClient{stream} |
|||
if err := x.ClientStream.SendMsg(in); err != nil { |
|||
return nil, err |
|||
} |
|||
if err := x.ClientStream.CloseSend(); err != nil { |
|||
return nil, err |
|||
} |
|||
return x, nil |
|||
} |
|||
|
|||
type SeaweedQueue_StreamReadClient interface { |
|||
Recv() (*ReadMessageResponse, error) |
|||
grpc.ClientStream |
|||
} |
|||
|
|||
type seaweedQueueStreamReadClient struct { |
|||
grpc.ClientStream |
|||
} |
|||
|
|||
func (x *seaweedQueueStreamReadClient) Recv() (*ReadMessageResponse, error) { |
|||
m := new(ReadMessageResponse) |
|||
if err := x.ClientStream.RecvMsg(m); err != nil { |
|||
return nil, err |
|||
} |
|||
return m, nil |
|||
} |
|||
|
|||
func (c *seaweedQueueClient) ConfigureTopic(ctx context.Context, in *ConfigureTopicRequest, opts ...grpc.CallOption) (*ConfigureTopicResponse, error) { |
|||
out := new(ConfigureTopicResponse) |
|||
err := grpc.Invoke(ctx, "/queue_pb.SeaweedQueue/ConfigureTopic", in, out, c.cc, opts...) |
|||
if err != nil { |
|||
return nil, err |
|||
} |
|||
return out, nil |
|||
} |
|||
|
|||
func (c *seaweedQueueClient) DeleteTopic(ctx context.Context, in *DeleteTopicRequest, opts ...grpc.CallOption) (*DeleteTopicResponse, error) { |
|||
out := new(DeleteTopicResponse) |
|||
err := grpc.Invoke(ctx, "/queue_pb.SeaweedQueue/DeleteTopic", in, out, c.cc, opts...) |
|||
if err != nil { |
|||
return nil, err |
|||
} |
|||
return out, nil |
|||
} |
|||
|
|||
// Server API for SeaweedQueue service
|
|||
|
|||
type SeaweedQueueServer interface { |
|||
StreamWrite(SeaweedQueue_StreamWriteServer) error |
|||
StreamRead(*ReadMessageRequest, SeaweedQueue_StreamReadServer) error |
|||
ConfigureTopic(context.Context, *ConfigureTopicRequest) (*ConfigureTopicResponse, error) |
|||
DeleteTopic(context.Context, *DeleteTopicRequest) (*DeleteTopicResponse, error) |
|||
} |
|||
|
|||
func RegisterSeaweedQueueServer(s *grpc.Server, srv SeaweedQueueServer) { |
|||
s.RegisterService(&_SeaweedQueue_serviceDesc, srv) |
|||
} |
|||
|
|||
func _SeaweedQueue_StreamWrite_Handler(srv interface{}, stream grpc.ServerStream) error { |
|||
return srv.(SeaweedQueueServer).StreamWrite(&seaweedQueueStreamWriteServer{stream}) |
|||
} |
|||
|
|||
type SeaweedQueue_StreamWriteServer interface { |
|||
Send(*WriteMessageResponse) error |
|||
Recv() (*WriteMessageRequest, error) |
|||
grpc.ServerStream |
|||
} |
|||
|
|||
type seaweedQueueStreamWriteServer struct { |
|||
grpc.ServerStream |
|||
} |
|||
|
|||
func (x *seaweedQueueStreamWriteServer) Send(m *WriteMessageResponse) error { |
|||
return x.ServerStream.SendMsg(m) |
|||
} |
|||
|
|||
func (x *seaweedQueueStreamWriteServer) Recv() (*WriteMessageRequest, error) { |
|||
m := new(WriteMessageRequest) |
|||
if err := x.ServerStream.RecvMsg(m); err != nil { |
|||
return nil, err |
|||
} |
|||
return m, nil |
|||
} |
|||
|
|||
func _SeaweedQueue_StreamRead_Handler(srv interface{}, stream grpc.ServerStream) error { |
|||
m := new(ReadMessageRequest) |
|||
if err := stream.RecvMsg(m); err != nil { |
|||
return err |
|||
} |
|||
return srv.(SeaweedQueueServer).StreamRead(m, &seaweedQueueStreamReadServer{stream}) |
|||
} |
|||
|
|||
type SeaweedQueue_StreamReadServer interface { |
|||
Send(*ReadMessageResponse) error |
|||
grpc.ServerStream |
|||
} |
|||
|
|||
type seaweedQueueStreamReadServer struct { |
|||
grpc.ServerStream |
|||
} |
|||
|
|||
func (x *seaweedQueueStreamReadServer) Send(m *ReadMessageResponse) error { |
|||
return x.ServerStream.SendMsg(m) |
|||
} |
|||
|
|||
func _SeaweedQueue_ConfigureTopic_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { |
|||
in := new(ConfigureTopicRequest) |
|||
if err := dec(in); err != nil { |
|||
return nil, err |
|||
} |
|||
if interceptor == nil { |
|||
return srv.(SeaweedQueueServer).ConfigureTopic(ctx, in) |
|||
} |
|||
info := &grpc.UnaryServerInfo{ |
|||
Server: srv, |
|||
FullMethod: "/queue_pb.SeaweedQueue/ConfigureTopic", |
|||
} |
|||
handler := func(ctx context.Context, req interface{}) (interface{}, error) { |
|||
return srv.(SeaweedQueueServer).ConfigureTopic(ctx, req.(*ConfigureTopicRequest)) |
|||
} |
|||
return interceptor(ctx, in, info, handler) |
|||
} |
|||
|
|||
func _SeaweedQueue_DeleteTopic_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { |
|||
in := new(DeleteTopicRequest) |
|||
if err := dec(in); err != nil { |
|||
return nil, err |
|||
} |
|||
if interceptor == nil { |
|||
return srv.(SeaweedQueueServer).DeleteTopic(ctx, in) |
|||
} |
|||
info := &grpc.UnaryServerInfo{ |
|||
Server: srv, |
|||
FullMethod: "/queue_pb.SeaweedQueue/DeleteTopic", |
|||
} |
|||
handler := func(ctx context.Context, req interface{}) (interface{}, error) { |
|||
return srv.(SeaweedQueueServer).DeleteTopic(ctx, req.(*DeleteTopicRequest)) |
|||
} |
|||
return interceptor(ctx, in, info, handler) |
|||
} |
|||
|
|||
var _SeaweedQueue_serviceDesc = grpc.ServiceDesc{ |
|||
ServiceName: "queue_pb.SeaweedQueue", |
|||
HandlerType: (*SeaweedQueueServer)(nil), |
|||
Methods: []grpc.MethodDesc{ |
|||
{ |
|||
MethodName: "ConfigureTopic", |
|||
Handler: _SeaweedQueue_ConfigureTopic_Handler, |
|||
}, |
|||
{ |
|||
MethodName: "DeleteTopic", |
|||
Handler: _SeaweedQueue_DeleteTopic_Handler, |
|||
}, |
|||
}, |
|||
Streams: []grpc.StreamDesc{ |
|||
{ |
|||
StreamName: "StreamWrite", |
|||
Handler: _SeaweedQueue_StreamWrite_Handler, |
|||
ServerStreams: true, |
|||
ClientStreams: true, |
|||
}, |
|||
{ |
|||
StreamName: "StreamRead", |
|||
Handler: _SeaweedQueue_StreamRead_Handler, |
|||
ServerStreams: true, |
|||
}, |
|||
}, |
|||
Metadata: "queue.proto", |
|||
} |
|||
|
|||
func init() { proto.RegisterFile("queue.proto", fileDescriptor0) } |
|||
|
|||
var fileDescriptor0 = []byte{ |
|||
// 429 bytes of a gzipped FileDescriptorProto
|
|||
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0x8c, 0x53, 0xcd, 0x6e, 0xd3, 0x40, |
|||
0x10, 0xae, 0x9b, 0xa6, 0x94, 0x49, 0x28, 0x68, 0xd2, 0xa2, 0x10, 0xd1, 0x36, 0x5a, 0x0e, 0x44, |
|||
0x20, 0x59, 0x15, 0xbc, 0x41, 0x03, 0x27, 0x68, 0x04, 0x0e, 0x08, 0x89, 0x8b, 0xb5, 0xb5, 0xa7, |
|||
0x95, 0x15, 0xb3, 0xeb, 0xee, 0x8e, 0xa9, 0x7a, 0xe2, 0x2d, 0x79, 0x1e, 0xe4, 0xb5, 0x5c, 0xdb, |
|||
0x34, 0xb1, 0x7a, 0xf3, 0xcc, 0x78, 0xe7, 0xfb, 0xd9, 0x6f, 0x61, 0x70, 0x9d, 0x53, 0x4e, 0x7e, |
|||
0x66, 0x34, 0x6b, 0xdc, 0x73, 0x45, 0x98, 0x5d, 0x88, 0x3f, 0x30, 0xfa, 0x61, 0x12, 0xa6, 0x73, |
|||
0xb2, 0x56, 0x5e, 0x51, 0x40, 0xd7, 0x39, 0x59, 0xc6, 0x03, 0xe8, 0xb3, 0xce, 0x92, 0x68, 0xec, |
|||
0x4d, 0xbd, 0xd9, 0xe3, 0xa0, 0x2c, 0xf0, 0x05, 0xec, 0xd1, 0x6f, 0x52, 0x1c, 0x2a, 0x3b, 0xde, |
|||
0x9e, 0x7a, 0xb3, 0x5e, 0xf0, 0xc8, 0xd5, 0x0b, 0x8b, 0xaf, 0xe0, 0x49, 0x26, 0x0d, 0x27, 0x9c, |
|||
0x68, 0x15, 0xae, 0xe8, 0x76, 0xdc, 0x9b, 0x7a, 0xb3, 0x61, 0x30, 0xbc, 0x6b, 0x7e, 0xa2, 0x5b, |
|||
0x44, 0xd8, 0x89, 0x25, 0xcb, 0xf1, 0x8e, 0x9b, 0xb9, 0x6f, 0x31, 0x87, 0x83, 0x36, 0x01, 0x9b, |
|||
0x69, 0x65, 0xa9, 0x60, 0x40, 0xc6, 0x68, 0x53, 0x31, 0x70, 0x05, 0x1e, 0xc2, 0xae, 0x8c, 0x56, |
|||
0x35, 0x7e, 0x5f, 0x46, 0xab, 0x85, 0x15, 0x1f, 0x01, 0x03, 0x92, 0xf1, 0x43, 0x45, 0x58, 0x96, |
|||
0xa6, 0x29, 0xc2, 0xd5, 0x0b, 0x2b, 0x7e, 0xc2, 0xa8, 0xb5, 0xa6, 0x93, 0x4a, 0x87, 0x19, 0x95, |
|||
0xce, 0x5e, 0x43, 0xe7, 0x0d, 0x1c, 0xce, 0xb5, 0xba, 0x4c, 0xae, 0x72, 0x43, 0xdf, 0x0a, 0x22, |
|||
0xdd, 0x2c, 0x4f, 0x60, 0xc0, 0x9c, 0x86, 0x96, 0x22, 0xad, 0xe2, 0x0a, 0x00, 0x98, 0xd3, 0x65, |
|||
0xd9, 0xc1, 0xd7, 0xf0, 0xb4, 0x36, 0x3c, 0xd2, 0xb9, 0x62, 0x07, 0xd7, 0x0f, 0xf6, 0xef, 0xda, |
|||
0xf3, 0xa2, 0x2b, 0x7c, 0x78, 0xfe, 0x3f, 0x70, 0x97, 0x2e, 0xf1, 0x06, 0xf0, 0x03, 0xa5, 0xc4, |
|||
0x0f, 0x60, 0x29, 0xde, 0xc2, 0xa8, 0xf5, 0x6f, 0xd7, 0xe2, 0x77, 0x7f, 0xb7, 0x61, 0xb8, 0x24, |
|||
0x79, 0x43, 0x14, 0x7f, 0x2d, 0xe2, 0x87, 0x01, 0x0c, 0x96, 0x6c, 0x48, 0xfe, 0x72, 0x01, 0xc0, |
|||
0x23, 0xbf, 0x4a, 0xa5, 0xbf, 0x26, 0x92, 0x93, 0xe3, 0x4d, 0xe3, 0x12, 0x54, 0x6c, 0xcd, 0xbc, |
|||
0x53, 0x0f, 0xcf, 0x01, 0xca, 0x9d, 0xc5, 0x45, 0xe2, 0xcb, 0xfa, 0xcc, 0xfd, 0x7c, 0x4c, 0x8e, |
|||
0x36, 0x4c, 0xab, 0x85, 0xa7, 0x1e, 0x7e, 0x87, 0xfd, 0xb6, 0x79, 0x78, 0x52, 0x1f, 0x5a, 0x7b, |
|||
0x9f, 0x93, 0xe9, 0xe6, 0x1f, 0xaa, 0xc5, 0xf8, 0x19, 0x06, 0x0d, 0xdf, 0x9a, 0x34, 0xef, 0x5b, |
|||
0xdf, 0xa4, 0xb9, 0xc6, 0x6c, 0xb1, 0x75, 0x76, 0x0c, 0xcf, 0x6c, 0xe9, 0xeb, 0xa5, 0xf5, 0xa3, |
|||
0x34, 0x21, 0xc5, 0x67, 0xe0, 0x2c, 0xfe, 0x52, 0xbc, 0xf6, 0x8b, 0x5d, 0xf7, 0xe8, 0xdf, 0xff, |
|||
0x0b, 0x00, 0x00, 0xff, 0xff, 0x7d, 0x3e, 0x14, 0xd8, 0x03, 0x04, 0x00, 0x00, |
|||
} |
|||
@ -0,0 +1,5 @@ |
|||
package pb |
|||
|
|||
const ( |
|||
AdminShellClient = "shell" |
|||
) |
|||
@ -1,23 +0,0 @@ |
|||
package weed_server |
|||
|
|||
import ( |
|||
"context" |
|||
|
|||
"github.com/chrislusf/seaweedfs/weed/pb/queue_pb" |
|||
) |
|||
|
|||
func (broker *MessageBroker) ConfigureTopic(context.Context, *queue_pb.ConfigureTopicRequest) (*queue_pb.ConfigureTopicResponse, error) { |
|||
panic("implement me") |
|||
} |
|||
|
|||
func (broker *MessageBroker) DeleteTopic(context.Context, *queue_pb.DeleteTopicRequest) (*queue_pb.DeleteTopicResponse, error) { |
|||
panic("implement me") |
|||
} |
|||
|
|||
func (broker *MessageBroker) StreamWrite(queue_pb.SeaweedQueue_StreamWriteServer) error { |
|||
panic("implement me") |
|||
} |
|||
|
|||
func (broker *MessageBroker) StreamRead(*queue_pb.ReadMessageRequest, queue_pb.SeaweedQueue_StreamReadServer) error { |
|||
panic("implement me") |
|||
} |
|||
@ -0,0 +1,113 @@ |
|||
package chunk_cache |
|||
|
|||
import ( |
|||
"sync" |
|||
|
|||
"github.com/chrislusf/seaweedfs/weed/glog" |
|||
"github.com/chrislusf/seaweedfs/weed/storage/needle" |
|||
) |
|||
|
|||
const ( |
|||
memCacheSizeLimit = 1024 * 1024 |
|||
onDiskCacheSizeLimit0 = memCacheSizeLimit |
|||
onDiskCacheSizeLimit1 = 4 * memCacheSizeLimit |
|||
) |
|||
|
|||
// a global cache for recently accessed file chunks
|
|||
type ChunkCache struct { |
|||
memCache *ChunkCacheInMemory |
|||
diskCaches []*OnDiskCacheLayer |
|||
sync.RWMutex |
|||
} |
|||
|
|||
func NewChunkCache(maxEntries int64, dir string, diskSizeMB int64) *ChunkCache { |
|||
|
|||
c := &ChunkCache{ |
|||
memCache: NewChunkCacheInMemory(maxEntries), |
|||
} |
|||
c.diskCaches = make([]*OnDiskCacheLayer, 3) |
|||
c.diskCaches[0] = NewOnDiskCacheLayer(dir, "c0_1", diskSizeMB/4, 4) |
|||
c.diskCaches[1] = NewOnDiskCacheLayer(dir, "c1_4", diskSizeMB/4, 4) |
|||
c.diskCaches[2] = NewOnDiskCacheLayer(dir, "cache", diskSizeMB/2, 4) |
|||
|
|||
return c |
|||
} |
|||
|
|||
func (c *ChunkCache) GetChunk(fileId string, chunkSize uint64) (data []byte) { |
|||
if c == nil { |
|||
return |
|||
} |
|||
|
|||
c.RLock() |
|||
defer c.RUnlock() |
|||
|
|||
return c.doGetChunk(fileId, chunkSize) |
|||
} |
|||
|
|||
func (c *ChunkCache) doGetChunk(fileId string, chunkSize uint64) (data []byte) { |
|||
|
|||
if chunkSize < memCacheSizeLimit { |
|||
if data = c.memCache.GetChunk(fileId); data != nil { |
|||
return data |
|||
} |
|||
} |
|||
|
|||
fid, err := needle.ParseFileIdFromString(fileId) |
|||
if err != nil { |
|||
glog.Errorf("failed to parse file id %s", fileId) |
|||
return nil |
|||
} |
|||
|
|||
for _, diskCache := range c.diskCaches { |
|||
data := diskCache.getChunk(fid.Key) |
|||
if len(data) != 0 { |
|||
return data |
|||
} |
|||
} |
|||
|
|||
return nil |
|||
|
|||
} |
|||
|
|||
func (c *ChunkCache) SetChunk(fileId string, data []byte) { |
|||
if c == nil { |
|||
return |
|||
} |
|||
c.Lock() |
|||
defer c.Unlock() |
|||
|
|||
c.doSetChunk(fileId, data) |
|||
} |
|||
|
|||
func (c *ChunkCache) doSetChunk(fileId string, data []byte) { |
|||
|
|||
if len(data) < memCacheSizeLimit { |
|||
c.memCache.SetChunk(fileId, data) |
|||
} |
|||
|
|||
fid, err := needle.ParseFileIdFromString(fileId) |
|||
if err != nil { |
|||
glog.Errorf("failed to parse file id %s", fileId) |
|||
return |
|||
} |
|||
|
|||
if len(data) < onDiskCacheSizeLimit0 { |
|||
c.diskCaches[0].setChunk(fid.Key, data) |
|||
} else if len(data) < onDiskCacheSizeLimit1 { |
|||
c.diskCaches[1].setChunk(fid.Key, data) |
|||
} else { |
|||
c.diskCaches[2].setChunk(fid.Key, data) |
|||
} |
|||
|
|||
} |
|||
|
|||
func (c *ChunkCache) Shutdown() { |
|||
if c == nil { |
|||
return |
|||
} |
|||
c.Lock() |
|||
defer c.Unlock() |
|||
for _, diskCache := range c.diskCaches { |
|||
diskCache.shutdown() |
|||
} |
|||
} |
|||
@ -0,0 +1,145 @@ |
|||
package chunk_cache |
|||
|
|||
import ( |
|||
"fmt" |
|||
"os" |
|||
"time" |
|||
|
|||
"github.com/syndtr/goleveldb/leveldb/opt" |
|||
|
|||
"github.com/chrislusf/seaweedfs/weed/glog" |
|||
"github.com/chrislusf/seaweedfs/weed/storage" |
|||
"github.com/chrislusf/seaweedfs/weed/storage/backend" |
|||
"github.com/chrislusf/seaweedfs/weed/storage/types" |
|||
"github.com/chrislusf/seaweedfs/weed/util" |
|||
) |
|||
|
|||
// This implements an on disk cache
|
|||
// The entries are an FIFO with a size limit
|
|||
|
|||
type ChunkCacheVolume struct { |
|||
DataBackend backend.BackendStorageFile |
|||
nm storage.NeedleMapper |
|||
fileName string |
|||
smallBuffer []byte |
|||
sizeLimit int64 |
|||
lastModTime time.Time |
|||
fileSize int64 |
|||
} |
|||
|
|||
func LoadOrCreateChunkCacheVolume(fileName string, preallocate int64) (*ChunkCacheVolume, error) { |
|||
|
|||
v := &ChunkCacheVolume{ |
|||
smallBuffer: make([]byte, types.NeedlePaddingSize), |
|||
fileName: fileName, |
|||
sizeLimit: preallocate, |
|||
} |
|||
|
|||
var err error |
|||
|
|||
if exists, canRead, canWrite, modTime, fileSize := util.CheckFile(v.fileName + ".dat"); exists { |
|||
if !canRead { |
|||
return nil, fmt.Errorf("cannot read cache file %s.dat", v.fileName) |
|||
} |
|||
if !canWrite { |
|||
return nil, fmt.Errorf("cannot write cache file %s.dat", v.fileName) |
|||
} |
|||
if dataFile, err := os.OpenFile(v.fileName+".dat", os.O_RDWR|os.O_CREATE, 0644); err != nil { |
|||
return nil, fmt.Errorf("cannot create cache file %s.dat: %v", v.fileName, err) |
|||
} else { |
|||
v.DataBackend = backend.NewDiskFile(dataFile) |
|||
v.lastModTime = modTime |
|||
v.fileSize = fileSize |
|||
} |
|||
} else { |
|||
if v.DataBackend, err = backend.CreateVolumeFile(v.fileName+".dat", preallocate, 0); err != nil { |
|||
return nil, fmt.Errorf("cannot create cache file %s.dat: %v", v.fileName, err) |
|||
} |
|||
v.lastModTime = time.Now() |
|||
} |
|||
|
|||
var indexFile *os.File |
|||
if indexFile, err = os.OpenFile(v.fileName+".idx", os.O_RDWR|os.O_CREATE, 0644); err != nil { |
|||
return nil, fmt.Errorf("cannot write cache index %s.idx: %v", v.fileName, err) |
|||
} |
|||
|
|||
glog.V(0).Infoln("loading leveldb", v.fileName+".ldb") |
|||
opts := &opt.Options{ |
|||
BlockCacheCapacity: 2 * 1024 * 1024, // default value is 8MiB
|
|||
WriteBuffer: 1 * 1024 * 1024, // default value is 4MiB
|
|||
CompactionTableSizeMultiplier: 10, // default value is 1
|
|||
} |
|||
if v.nm, err = storage.NewLevelDbNeedleMap(v.fileName+".ldb", indexFile, opts); err != nil { |
|||
return nil, fmt.Errorf("loading leveldb %s error: %v", v.fileName+".ldb", err) |
|||
} |
|||
|
|||
return v, nil |
|||
|
|||
} |
|||
|
|||
func (v *ChunkCacheVolume) Shutdown() { |
|||
if v.DataBackend != nil { |
|||
v.DataBackend.Close() |
|||
v.DataBackend = nil |
|||
} |
|||
if v.nm != nil { |
|||
v.nm.Close() |
|||
v.nm = nil |
|||
} |
|||
} |
|||
|
|||
func (v *ChunkCacheVolume) destroy() { |
|||
v.Shutdown() |
|||
os.Remove(v.fileName + ".dat") |
|||
os.Remove(v.fileName + ".idx") |
|||
os.RemoveAll(v.fileName + ".ldb") |
|||
} |
|||
|
|||
func (v *ChunkCacheVolume) Reset() (*ChunkCacheVolume, error) { |
|||
v.destroy() |
|||
return LoadOrCreateChunkCacheVolume(v.fileName, v.sizeLimit) |
|||
} |
|||
|
|||
func (v *ChunkCacheVolume) GetNeedle(key types.NeedleId) ([]byte, error) { |
|||
|
|||
nv, ok := v.nm.Get(key) |
|||
if !ok { |
|||
return nil, storage.ErrorNotFound |
|||
} |
|||
data := make([]byte, nv.Size) |
|||
if readSize, readErr := v.DataBackend.ReadAt(data, nv.Offset.ToAcutalOffset()); readErr != nil { |
|||
return nil, fmt.Errorf("read %s.dat [%d,%d): %v", |
|||
v.fileName, nv.Offset.ToAcutalOffset(), nv.Offset.ToAcutalOffset()+int64(nv.Size), readErr) |
|||
} else { |
|||
if readSize != int(nv.Size) { |
|||
return nil, fmt.Errorf("read %d, expected %d", readSize, nv.Size) |
|||
} |
|||
} |
|||
|
|||
return data, nil |
|||
} |
|||
|
|||
func (v *ChunkCacheVolume) WriteNeedle(key types.NeedleId, data []byte) error { |
|||
|
|||
offset := v.fileSize |
|||
|
|||
written, err := v.DataBackend.WriteAt(data, offset) |
|||
if err != nil { |
|||
return err |
|||
} else if written != len(data) { |
|||
return fmt.Errorf("partial written %d, expected %d", written, len(data)) |
|||
} |
|||
|
|||
v.fileSize += int64(written) |
|||
extraSize := written % types.NeedlePaddingSize |
|||
if extraSize != 0 { |
|||
v.DataBackend.WriteAt(v.smallBuffer[:types.NeedlePaddingSize-extraSize], offset+int64(written)) |
|||
v.fileSize += int64(types.NeedlePaddingSize - extraSize) |
|||
} |
|||
|
|||
if err := v.nm.Put(key, types.ToOffset(offset), uint32(len(data))); err != nil { |
|||
glog.V(4).Infof("failed to save in needle map %d: %v", key, err) |
|||
} |
|||
|
|||
return nil |
|||
} |
|||
@ -0,0 +1,59 @@ |
|||
package chunk_cache |
|||
|
|||
import ( |
|||
"bytes" |
|||
"fmt" |
|||
"io/ioutil" |
|||
"math/rand" |
|||
"os" |
|||
"testing" |
|||
) |
|||
|
|||
func TestOnDisk(t *testing.T) { |
|||
|
|||
tmpDir, _ := ioutil.TempDir("", "c") |
|||
defer os.RemoveAll(tmpDir) |
|||
|
|||
totalDiskSizeMb := int64(32) |
|||
|
|||
cache := NewChunkCache(0, tmpDir, totalDiskSizeMb) |
|||
|
|||
writeCount := 5 |
|||
type test_data struct { |
|||
data []byte |
|||
fileId string |
|||
size uint64 |
|||
} |
|||
testData := make([]*test_data, writeCount) |
|||
for i := 0; i < writeCount; i++ { |
|||
buff := make([]byte, 1024*1024) |
|||
rand.Read(buff) |
|||
testData[i] = &test_data{ |
|||
data: buff, |
|||
fileId: fmt.Sprintf("1,%daabbccdd", i+1), |
|||
size: uint64(len(buff)), |
|||
} |
|||
cache.SetChunk(testData[i].fileId, testData[i].data) |
|||
} |
|||
|
|||
for i := 0; i < writeCount; i++ { |
|||
data := cache.GetChunk(testData[i].fileId, testData[i].size) |
|||
if bytes.Compare(data, testData[i].data) != 0 { |
|||
t.Errorf("failed to write to and read from cache: %d", i) |
|||
} |
|||
} |
|||
|
|||
cache.Shutdown() |
|||
|
|||
cache = NewChunkCache(0, tmpDir, totalDiskSizeMb) |
|||
|
|||
for i := 0; i < writeCount; i++ { |
|||
data := cache.GetChunk(testData[i].fileId, testData[i].size) |
|||
if bytes.Compare(data, testData[i].data) != 0 { |
|||
t.Errorf("failed to write to and read from cache: %d", i) |
|||
} |
|||
} |
|||
|
|||
cache.Shutdown() |
|||
|
|||
} |
|||
@ -0,0 +1,89 @@ |
|||
package chunk_cache |
|||
|
|||
import ( |
|||
"fmt" |
|||
"path" |
|||
"sort" |
|||
|
|||
"github.com/chrislusf/seaweedfs/weed/glog" |
|||
"github.com/chrislusf/seaweedfs/weed/storage" |
|||
"github.com/chrislusf/seaweedfs/weed/storage/types" |
|||
) |
|||
|
|||
type OnDiskCacheLayer struct { |
|||
diskCaches []*ChunkCacheVolume |
|||
} |
|||
|
|||
func NewOnDiskCacheLayer(dir, namePrefix string, diskSizeMB int64, segmentCount int) *OnDiskCacheLayer{ |
|||
|
|||
volumeCount, volumeSize := int(diskSizeMB/30000), int64(30000) |
|||
if volumeCount < segmentCount { |
|||
volumeCount, volumeSize = segmentCount, diskSizeMB/int64(segmentCount) |
|||
} |
|||
|
|||
c := &OnDiskCacheLayer{} |
|||
for i := 0; i < volumeCount; i++ { |
|||
fileName := path.Join(dir, fmt.Sprintf("%s_%d", namePrefix, i)) |
|||
diskCache, err := LoadOrCreateChunkCacheVolume(fileName, volumeSize*1024*1024) |
|||
if err != nil { |
|||
glog.Errorf("failed to add cache %s : %v", fileName, err) |
|||
} else { |
|||
c.diskCaches = append(c.diskCaches, diskCache) |
|||
} |
|||
} |
|||
|
|||
// keep newest cache to the front
|
|||
sort.Slice(c.diskCaches, func(i, j int) bool { |
|||
return c.diskCaches[i].lastModTime.After(c.diskCaches[j].lastModTime) |
|||
}) |
|||
|
|||
return c |
|||
} |
|||
|
|||
func (c *OnDiskCacheLayer) setChunk(needleId types.NeedleId, data []byte) { |
|||
|
|||
if c.diskCaches[0].fileSize+int64(len(data)) > c.diskCaches[0].sizeLimit { |
|||
t, resetErr := c.diskCaches[len(c.diskCaches)-1].Reset() |
|||
if resetErr != nil { |
|||
glog.Errorf("failed to reset cache file %s", c.diskCaches[len(c.diskCaches)-1].fileName) |
|||
return |
|||
} |
|||
for i := len(c.diskCaches) - 1; i > 0; i-- { |
|||
c.diskCaches[i] = c.diskCaches[i-1] |
|||
} |
|||
c.diskCaches[0] = t |
|||
} |
|||
|
|||
c.diskCaches[0].WriteNeedle(needleId, data) |
|||
|
|||
} |
|||
|
|||
func (c *OnDiskCacheLayer) getChunk(needleId types.NeedleId) (data []byte){ |
|||
|
|||
var err error |
|||
|
|||
for _, diskCache := range c.diskCaches { |
|||
data, err = diskCache.GetNeedle(needleId) |
|||
if err == storage.ErrorNotFound { |
|||
continue |
|||
} |
|||
if err != nil { |
|||
glog.Errorf("failed to read cache file %s id %d", diskCache.fileName, needleId) |
|||
continue |
|||
} |
|||
if len(data) != 0 { |
|||
return |
|||
} |
|||
} |
|||
|
|||
return nil |
|||
|
|||
} |
|||
|
|||
func (c *OnDiskCacheLayer) shutdown(){ |
|||
|
|||
for _, diskCache := range c.diskCaches { |
|||
diskCache.Shutdown() |
|||
} |
|||
|
|||
} |
|||
@ -0,0 +1,40 @@ |
|||
package log_buffer |
|||
|
|||
import "time" |
|||
|
|||
type MemBuffer struct { |
|||
buf []byte |
|||
startTime time.Time |
|||
stopTime time.Time |
|||
} |
|||
|
|||
type SealedBuffers struct { |
|||
buffers []*MemBuffer |
|||
} |
|||
|
|||
func newSealedBuffers(size int) *SealedBuffers { |
|||
sbs := &SealedBuffers{} |
|||
|
|||
sbs.buffers = make([]*MemBuffer, size) |
|||
for i := 0; i < size; i++ { |
|||
sbs.buffers[i] = &MemBuffer{ |
|||
buf: make([]byte, BufferSize), |
|||
} |
|||
} |
|||
|
|||
return sbs |
|||
} |
|||
|
|||
func (sbs *SealedBuffers) SealBuffer(startTime, stopTime time.Time, buf []byte) (newBuf []byte) { |
|||
oldMemBuffer := sbs.buffers[0] |
|||
size := len(sbs.buffers) |
|||
for i := 0; i < size-1; i++ { |
|||
sbs.buffers[i].buf = sbs.buffers[i+1].buf |
|||
sbs.buffers[i].startTime = sbs.buffers[i+1].startTime |
|||
sbs.buffers[i].stopTime = sbs.buffers[i+1].stopTime |
|||
} |
|||
sbs.buffers[size-1].buf = buf |
|||
sbs.buffers[size-1].startTime = startTime |
|||
sbs.buffers[size-1].stopTime = stopTime |
|||
return oldMemBuffer.buf |
|||
} |
|||
@ -0,0 +1,25 @@ |
|||
package util |
|||
|
|||
import ( |
|||
"net" |
|||
|
|||
"github.com/chrislusf/seaweedfs/weed/glog" |
|||
) |
|||
|
|||
func DetectedHostAddress() string { |
|||
addrs, err := net.InterfaceAddrs() |
|||
if err != nil { |
|||
glog.V(0).Infof("failed to detect ip address: %v", err) |
|||
return "" |
|||
} |
|||
|
|||
for _, a := range addrs { |
|||
if ipnet, ok := a.(*net.IPNet); ok && !ipnet.IP.IsLoopback() { |
|||
if ipnet.IP.To4() != nil { |
|||
return ipnet.IP.String() |
|||
} |
|||
} |
|||
} |
|||
|
|||
return "localhost" |
|||
} |
|||
Some files were not shown because too many files changed in this diff
Write
Preview
Loading…
Cancel
Save
Reference in new issue