From 90d576affe398b372695a767b49ddc03cacc1be4 Mon Sep 17 00:00:00 2001
From: Chris Lu <chris.lu@gmail.com>
Date: Tue, 7 Apr 2020 01:05:24 -0700
Subject: [PATCH 01/81] disable meta data change event logging for now.

---
 weed/filer2/filer_notify.go | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/weed/filer2/filer_notify.go b/weed/filer2/filer_notify.go
index f85ee1db4..c57631466 100644
--- a/weed/filer2/filer_notify.go
+++ b/weed/filer2/filer_notify.go
@@ -45,7 +45,9 @@ func (f *Filer) NotifyUpdateEvent(oldEntry, newEntry *Entry, deleteChunks bool)
 		notification.Queue.SendMessage(fullpath, eventNotification)
 	}
 
-	f.logMetaEvent(time.Now(), fullpath, eventNotification)
+	if false {
+		f.logMetaEvent(time.Now(), fullpath, eventNotification)
+	}
 
 }
 

From 54debdc6f73678f7e0988e636ee9424d2489f6be Mon Sep 17 00:00:00 2001
From: Chris Lu <chris.lu@gmail.com>
Date: Tue, 7 Apr 2020 01:30:53 -0700
Subject: [PATCH 02/81] filer: fix configuration settings

---
 weed/server/filer_server.go | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/weed/server/filer_server.go b/weed/server/filer_server.go
index 22bbd2a03..106066bb9 100644
--- a/weed/server/filer_server.go
+++ b/weed/server/filer_server.go
@@ -91,10 +91,10 @@ func NewFilerServer(defaultMux, readonlyMux *http.ServeMux, option *FilerOption)
 	util.LoadConfiguration("notification", false)
 
 	fs.option.recursiveDelete = v.GetBool("filer.options.recursive_delete")
-	v.Set("filer.option.buckets_folder", "/buckets")
-	v.Set("filer.option.queues_folder", "/queues")
-	fs.filer.DirBucketsPath = v.GetString("filer.option.buckets_folder")
-	fs.filer.DirQueuesPath = v.GetString("filer.option.queues_folder")
+	v.Set("filer.options.buckets_folder", "/buckets")
+	v.Set("filer.options.queues_folder", "/queues")
+	fs.filer.DirBucketsPath = v.GetString("filer.options.buckets_folder")
+	fs.filer.DirQueuesPath = v.GetString("filer.options.queues_folder")
 	fs.filer.LoadConfiguration(v)
 
 	notification.LoadConfiguration(v, "notification.")

From 745f5d2a257785cd165111f2b20992d0422b90c8 Mon Sep 17 00:00:00 2001
From: Chris Lu <chris.lu@gmail.com>
Date: Tue, 7 Apr 2020 01:58:48 -0700
Subject: [PATCH 03/81] fix setting default value

---
 weed/server/filer_server.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/weed/server/filer_server.go b/weed/server/filer_server.go
index 106066bb9..52dfbaf88 100644
--- a/weed/server/filer_server.go
+++ b/weed/server/filer_server.go
@@ -91,8 +91,8 @@ func NewFilerServer(defaultMux, readonlyMux *http.ServeMux, option *FilerOption)
 	util.LoadConfiguration("notification", false)
 
 	fs.option.recursiveDelete = v.GetBool("filer.options.recursive_delete")
-	v.Set("filer.options.buckets_folder", "/buckets")
-	v.Set("filer.options.queues_folder", "/queues")
+	v.SetDefault("filer.options.buckets_folder", "/buckets")
+	v.SetDefault("filer.options.queues_folder", "/queues")
 	fs.filer.DirBucketsPath = v.GetString("filer.options.buckets_folder")
 	fs.filer.DirQueuesPath = v.GetString("filer.options.queues_folder")
 	fs.filer.LoadConfiguration(v)

From 4c498e73de47c47e52bba0cab55c2c50ebc9929e Mon Sep 17 00:00:00 2001
From: Chris Lu <chris.lu@gmail.com>
Date: Tue, 7 Apr 2020 17:49:00 -0700
Subject: [PATCH 04/81] filer replication: add s3 endpoint

---
 weed/command/scaffold.go                | 1 +
 weed/replication/sink/s3sink/s3_sink.go | 9 +++++++--
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/weed/command/scaffold.go b/weed/command/scaffold.go
index e391c23ea..72a36bb54 100644
--- a/weed/command/scaffold.go
+++ b/weed/command/scaffold.go
@@ -260,6 +260,7 @@ aws_secret_access_key = ""     # if empty, loads from the shared credentials fil
 region = "us-east-2"
 bucket = "your_bucket_name"    # an existing bucket
 directory = "/"                # destination directory
+endpoint = ""
 
 [sink.google_cloud_storage]
 # read credentials doc at https://cloud.google.com/docs/authentication/getting-started
diff --git a/weed/replication/sink/s3sink/s3_sink.go b/weed/replication/sink/s3sink/s3_sink.go
index 5dbc3fdb7..d7af105b8 100644
--- a/weed/replication/sink/s3sink/s3_sink.go
+++ b/weed/replication/sink/s3sink/s3_sink.go
@@ -25,6 +25,7 @@ type S3Sink struct {
 	region      string
 	bucket      string
 	dir         string
+	endpoint    string
 	filerSource *source.FilerSource
 }
 
@@ -44,12 +45,14 @@ func (s3sink *S3Sink) Initialize(configuration util.Configuration, prefix string
 	glog.V(0).Infof("sink.s3.region: %v", configuration.GetString(prefix+"region"))
 	glog.V(0).Infof("sink.s3.bucket: %v", configuration.GetString(prefix+"bucket"))
 	glog.V(0).Infof("sink.s3.directory: %v", configuration.GetString(prefix+"directory"))
+	glog.V(0).Infof("sink.s3.endpoint: %v", configuration.GetString(prefix+"endpoint"))
 	return s3sink.initialize(
 		configuration.GetString(prefix+"aws_access_key_id"),
 		configuration.GetString(prefix+"aws_secret_access_key"),
 		configuration.GetString(prefix+"region"),
 		configuration.GetString(prefix+"bucket"),
 		configuration.GetString(prefix+"directory"),
+		configuration.GetString(prefix+"endpoint"),
 	)
 }
 
@@ -57,13 +60,15 @@ func (s3sink *S3Sink) SetSourceFiler(s *source.FilerSource) {
 	s3sink.filerSource = s
 }
 
-func (s3sink *S3Sink) initialize(awsAccessKeyId, awsSecretAccessKey, region, bucket, dir string) error {
+func (s3sink *S3Sink) initialize(awsAccessKeyId, awsSecretAccessKey, region, bucket, dir, endpoint string) error {
 	s3sink.region = region
 	s3sink.bucket = bucket
 	s3sink.dir = dir
+	s3sink.endpoint = endpoint
 
 	config := &aws.Config{
-		Region: aws.String(s3sink.region),
+		Region:   aws.String(s3sink.region),
+		Endpoint: aws.String(s3sink.endpoint),
 	}
 	if awsAccessKeyId != "" && awsSecretAccessKey != "" {
 		config.Credentials = credentials.NewStaticCredentials(awsAccessKeyId, awsSecretAccessKey, "")

From 2d7d4b1006a8cd1f7af6244672541e9ac16bfac9 Mon Sep 17 00:00:00 2001
From: Chris Lu <chris.lu@gmail.com>
Date: Wed, 8 Apr 2020 07:18:26 -0700
Subject: [PATCH 05/81] add tool to see log entries

---
 unmaintained/see_log_entry/see_log_entry.go | 74 +++++++++++++++++++++
 1 file changed, 74 insertions(+)
 create mode 100644 unmaintained/see_log_entry/see_log_entry.go

diff --git a/unmaintained/see_log_entry/see_log_entry.go b/unmaintained/see_log_entry/see_log_entry.go
new file mode 100644
index 000000000..b7d724344
--- /dev/null
+++ b/unmaintained/see_log_entry/see_log_entry.go
@@ -0,0 +1,74 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"io"
+	"log"
+	"os"
+
+	"github.com/golang/protobuf/proto"
+
+	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
+	"github.com/chrislusf/seaweedfs/weed/util"
+)
+
+var (
+	logdataFile = flag.String("logdata", "", "log data file saved under /.meta/log/...")
+)
+
+func main() {
+	flag.Parse()
+
+	dst, err := os.OpenFile(*logdataFile, os.O_RDONLY, 0644)
+	if err != nil {
+		log.Fatalf("failed to open %s: %v", *logdataFile, err)
+	}
+	defer dst.Close()
+
+	err = walkLogEntryFile(dst)
+	if err != nil {
+		log.Fatalf("failed to visit %s: %v", *logdataFile, err)
+	}
+
+}
+
+func walkLogEntryFile(dst *os.File) error {
+
+	sizeBuf := make([]byte, 4)
+
+	for {
+		if n, err := dst.Read(sizeBuf); n != 4 {
+			if err == io.EOF {
+				return nil
+			}
+			return err
+		}
+
+		size := util.BytesToUint32(sizeBuf)
+
+		data := make([]byte, int(size))
+
+		if n, err := dst.Read(data); n != len(data) {
+			return err
+		}
+
+		logEntry := &filer_pb.LogEntry{}
+		err := proto.Unmarshal(data, logEntry)
+		if err != nil {
+			log.Printf("unexpected unmarshal filer_pb.LogEntry: %v", err)
+			return nil
+		}
+
+		event := &filer_pb.FullEventNotification{}
+		err = proto.Unmarshal(logEntry.Data, event)
+		if err != nil {
+			log.Printf("unexpected unmarshal filer_pb.FullEventNotification: %v", err)
+			return nil
+		}
+
+		fmt.Printf("event: %+v\n", event)
+
+	}
+
+}

From ec2eb8bc4804f9b880f256a55e3cbfc0923b6a29 Mon Sep 17 00:00:00 2001
From: Chris Lu <chris.lu@gmail.com>
Date: Wed, 8 Apr 2020 08:12:00 -0700
Subject: [PATCH 06/81] add If-None-Match and If-Modified-Since

fix https://github.com/chrislusf/seaweedfs/issues/1269
---
 other/java/client/src/main/proto/filer.proto  |   1 +
 weed/filer2/entry.go                          |   1 +
 weed/filer2/entry_codec.go                    |   6 +
 weed/filer2/filechunks.go                     |  16 +-
 weed/pb/filer.proto                           |   1 +
 weed/pb/filer_pb/filer.pb.go                  | 246 +++++++++---------
 weed/replication/sink/filersink/filer_sink.go |   4 +-
 weed/s3api/filer_multipart.go                 |   4 +-
 weed/s3api/s3api_objects_list_handlers.go     |   2 +-
 weed/server/filer_server_handlers_read.go     |  21 +-
 weed/server/filer_server_handlers_write.go    |  18 +-
 .../filer_server_handlers_write_autochunk.go  |   8 +-
 12 files changed, 193 insertions(+), 135 deletions(-)

diff --git a/other/java/client/src/main/proto/filer.proto b/other/java/client/src/main/proto/filer.proto
index e504e4f84..8f8176ff4 100644
--- a/other/java/client/src/main/proto/filer.proto
+++ b/other/java/client/src/main/proto/filer.proto
@@ -123,6 +123,7 @@ message FuseAttributes {
     string user_name = 11; // for hdfs
     repeated string group_name = 12; // for hdfs
     string symlink_target = 13;
+    bytes md5 = 14;
 }
 
 message CreateEntryRequest {
diff --git a/weed/filer2/entry.go b/weed/filer2/entry.go
index ef6c8f9a6..6dff99af9 100644
--- a/weed/filer2/entry.go
+++ b/weed/filer2/entry.go
@@ -21,6 +21,7 @@ type Attr struct {
 	UserName      string
 	GroupNames    []string
 	SymlinkTarget string
+	Md5           []byte
 }
 
 func (attr Attr) IsDirectory() bool {
diff --git a/weed/filer2/entry_codec.go b/weed/filer2/entry_codec.go
index 3a2dc6134..47c911011 100644
--- a/weed/filer2/entry_codec.go
+++ b/weed/filer2/entry_codec.go
@@ -52,6 +52,7 @@ func EntryAttributeToPb(entry *Entry) *filer_pb.FuseAttributes {
 		UserName:      entry.Attr.UserName,
 		GroupName:     entry.Attr.GroupNames,
 		SymlinkTarget: entry.Attr.SymlinkTarget,
+		Md5:           entry.Attr.Md5,
 	}
 }
 
@@ -71,6 +72,7 @@ func PbToEntryAttribute(attr *filer_pb.FuseAttributes) Attr {
 	t.UserName = attr.UserName
 	t.GroupNames = attr.GroupName
 	t.SymlinkTarget = attr.SymlinkTarget
+	t.Md5 = attr.Md5
 
 	return t
 }
@@ -93,6 +95,10 @@ func EqualEntry(a, b *Entry) bool {
 		return false
 	}
 
+	if !bytes.Equal(a.Md5, b.Md5) {
+		return false
+	}
+
 	for i := 0; i < len(a.Chunks); i++ {
 		if !proto.Equal(a.Chunks[i], b.Chunks[i]) {
 			return false
diff --git a/weed/filer2/filechunks.go b/weed/filer2/filechunks.go
index 0c93c389b..e8e4c305b 100644
--- a/weed/filer2/filechunks.go
+++ b/weed/filer2/filechunks.go
@@ -20,7 +20,21 @@ func TotalSize(chunks []*filer_pb.FileChunk) (size uint64) {
 	return
 }
 
-func ETag(chunks []*filer_pb.FileChunk) (etag string) {
+func ETag(entry *filer_pb.Entry) (etag string) {
+	if entry.Attributes == nil || entry.Attributes.Md5 == nil {
+		ETagChunks(entry.Chunks)
+	}
+	return fmt.Sprintf("%x", entry.Attributes.Md5)
+}
+
+func ETagEntry(entry *Entry) (etag string) {
+	if entry.Attr.Md5 == nil {
+		ETagChunks(entry.Chunks)
+	}
+	return fmt.Sprintf("%x", entry.Attr.Md5)
+}
+
+func ETagChunks(chunks []*filer_pb.FileChunk) (etag string) {
 	if len(chunks) == 1 {
 		return chunks[0].ETag
 	}
diff --git a/weed/pb/filer.proto b/weed/pb/filer.proto
index e504e4f84..8f8176ff4 100644
--- a/weed/pb/filer.proto
+++ b/weed/pb/filer.proto
@@ -123,6 +123,7 @@ message FuseAttributes {
     string user_name = 11; // for hdfs
     repeated string group_name = 12; // for hdfs
     string symlink_target = 13;
+    bytes md5 = 14;
 }
 
 message CreateEntryRequest {
diff --git a/weed/pb/filer_pb/filer.pb.go b/weed/pb/filer_pb/filer.pb.go
index cd1f657af..814856593 100644
--- a/weed/pb/filer_pb/filer.pb.go
+++ b/weed/pb/filer_pb/filer.pb.go
@@ -415,6 +415,7 @@ type FuseAttributes struct {
 	UserName      string   `protobuf:"bytes,11,opt,name=user_name,json=userName" json:"user_name,omitempty"`
 	GroupName     []string `protobuf:"bytes,12,rep,name=group_name,json=groupName" json:"group_name,omitempty"`
 	SymlinkTarget string   `protobuf:"bytes,13,opt,name=symlink_target,json=symlinkTarget" json:"symlink_target,omitempty"`
+	Md5           []byte   `protobuf:"bytes,14,opt,name=md5,proto3" json:"md5,omitempty"`
 }
 
 func (m *FuseAttributes) Reset()                    { *m = FuseAttributes{} }
@@ -513,6 +514,13 @@ func (m *FuseAttributes) GetSymlinkTarget() string {
 	return ""
 }
 
+func (m *FuseAttributes) GetMd5() []byte {
+	if m != nil {
+		return m.Md5
+	}
+	return nil
+}
+
 type CreateEntryRequest struct {
 	Directory string `protobuf:"bytes,1,opt,name=directory" json:"directory,omitempty"`
 	Entry     *Entry `protobuf:"bytes,2,opt,name=entry" json:"entry,omitempty"`
@@ -1707,125 +1715,125 @@ var _SeaweedFiler_serviceDesc = grpc.ServiceDesc{
 func init() { proto.RegisterFile("filer.proto", fileDescriptor0) }
 
 var fileDescriptor0 = []byte{
-	// 1909 bytes of a gzipped FileDescriptorProto
+	// 1919 bytes of a gzipped FileDescriptorProto
 	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xb4, 0x58, 0x5f, 0x6f, 0xdc, 0xc6,
 	0x11, 0x37, 0xef, 0x74, 0x7f, 0x38, 0x77, 0x67, 0x4b, 0x7b, 0xb2, 0x73, 0x3e, 0x4b, 0xb6, 0x42,
-	0xd7, 0xa9, 0x0b, 0x1b, 0xaa, 0xa1, 0xe6, 0x21, 0x69, 0xda, 0x07, 0x5b, 0x96, 0x52, 0x37, 0xb6,
-	0xe2, 0x52, 0x76, 0x91, 0xa2, 0x40, 0x09, 0x8a, 0x5c, 0xdd, 0x6d, 0xc5, 0x23, 0x99, 0xdd, 0xa5,
-	0xfe, 0xe4, 0xad, 0x5f, 0xa3, 0x40, 0x1f, 0xfa, 0x1d, 0xfa, 0x58, 0xf4, 0xa5, 0x28, 0xd0, 0xcf,
-	0xd1, 0xc7, 0x3e, 0xf4, 0x33, 0x14, 0x3b, 0x4b, 0xf2, 0x96, 0xc7, 0x93, 0x94, 0x20, 0xc8, 0x1b,
-	0x77, 0x66, 0x76, 0x76, 0x76, 0xfe, 0xfc, 0x66, 0x96, 0xd0, 0x3b, 0x66, 0x11, 0xe5, 0xdb, 0x29,
-	0x4f, 0x64, 0x42, 0xba, 0xb8, 0xf0, 0xd2, 0x23, 0xe7, 0x4b, 0xb8, 0xf7, 0x3a, 0x49, 0x4e, 0xb2,
-	0xf4, 0x25, 0xe3, 0x34, 0x90, 0x09, 0xbf, 0xd8, 0x8b, 0x25, 0xbf, 0x70, 0xe9, 0xd7, 0x19, 0x15,
-	0x92, 0x6c, 0x80, 0x1d, 0x16, 0x8c, 0x91, 0xb5, 0x65, 0x3d, 0xb6, 0xdd, 0x39, 0x81, 0x10, 0x58,
-	0x89, 0xfd, 0x19, 0x1d, 0x35, 0x90, 0x81, 0xdf, 0xce, 0x1e, 0x6c, 0x2c, 0x57, 0x28, 0xd2, 0x24,
-	0x16, 0x94, 0x3c, 0x82, 0x16, 0x55, 0x04, 0xd4, 0xd6, 0xdb, 0xb9, 0xb5, 0x5d, 0x98, 0xb2, 0xad,
-	0xe5, 0x34, 0xd7, 0xf9, 0x87, 0x05, 0xe4, 0x35, 0x13, 0x52, 0x11, 0x19, 0x15, 0xdf, 0xce, 0x9e,
-	0x3b, 0xd0, 0x4e, 0x39, 0x3d, 0x66, 0xe7, 0xb9, 0x45, 0xf9, 0x8a, 0x3c, 0x85, 0x35, 0x21, 0x7d,
-	0x2e, 0xf7, 0x79, 0x32, 0xdb, 0x67, 0x11, 0x3d, 0x50, 0x46, 0x37, 0x51, 0xa4, 0xce, 0x20, 0xdb,
-	0x40, 0x58, 0x1c, 0x44, 0x99, 0x60, 0xa7, 0xf4, 0xb0, 0xe0, 0x8e, 0x56, 0xb6, 0xac, 0xc7, 0x5d,
-	0x77, 0x09, 0x87, 0xac, 0x43, 0x2b, 0x62, 0x33, 0x26, 0x47, 0xad, 0x2d, 0xeb, 0xf1, 0xc0, 0xd5,
-	0x0b, 0xe7, 0x17, 0x30, 0xac, 0xd8, 0xff, 0xdd, 0xae, 0xff, 0x97, 0x06, 0xb4, 0x90, 0x50, 0xfa,
-	0xd8, 0x9a, 0xfb, 0x98, 0x7c, 0x08, 0x7d, 0x26, 0xbc, 0xb9, 0x23, 0x1a, 0x68, 0x5b, 0x8f, 0x89,
-	0xd2, 0xe7, 0xe4, 0x09, 0xb4, 0x83, 0x69, 0x16, 0x9f, 0x88, 0x51, 0x73, 0xab, 0xf9, 0xb8, 0xb7,
-	0x33, 0x9c, 0x1f, 0xa4, 0x2e, 0xba, 0xab, 0x78, 0x6e, 0x2e, 0x42, 0x3e, 0x01, 0xf0, 0xa5, 0xe4,
-	0xec, 0x28, 0x93, 0x54, 0xe0, 0x4d, 0x7b, 0x3b, 0x23, 0x63, 0x43, 0x26, 0xe8, 0xf3, 0x92, 0xef,
-	0x1a, 0xb2, 0xe4, 0x53, 0xe8, 0xd2, 0x73, 0x49, 0xe3, 0x90, 0x86, 0xa3, 0x16, 0x1e, 0xb4, 0xb9,
-	0x70, 0xa3, 0xed, 0xbd, 0x9c, 0xaf, 0xef, 0x57, 0x8a, 0x8f, 0x3f, 0x83, 0x41, 0x85, 0x45, 0x56,
-	0xa1, 0x79, 0x42, 0x8b, 0xa8, 0xaa, 0x4f, 0xe5, 0xd9, 0x53, 0x3f, 0xca, 0x74, 0x82, 0xf5, 0x5d,
-	0xbd, 0xf8, 0x79, 0xe3, 0x13, 0xcb, 0x79, 0x09, 0xf6, 0x7e, 0x16, 0x45, 0xe5, 0xc6, 0x90, 0xf1,
-	0x62, 0x63, 0xc8, 0xf8, 0xdc, 0xcb, 0x8d, 0x2b, 0xbd, 0xfc, 0x77, 0x0b, 0xd6, 0xf6, 0x4e, 0x69,
-	0x2c, 0x0f, 0x12, 0xc9, 0x8e, 0x59, 0xe0, 0x4b, 0x96, 0xc4, 0xe4, 0x29, 0xd8, 0x49, 0x14, 0x7a,
-	0x57, 0x86, 0xa9, 0x9b, 0x44, 0xb9, 0xd5, 0x4f, 0xc1, 0x8e, 0xe9, 0x99, 0x77, 0xe5, 0x71, 0xdd,
-	0x98, 0x9e, 0x69, 0xe9, 0x87, 0x30, 0x08, 0x69, 0x44, 0x25, 0xf5, 0xca, 0xe8, 0xa8, 0xd0, 0xf5,
-	0x35, 0x71, 0x57, 0x87, 0xe3, 0x23, 0xb8, 0xa5, 0x54, 0xa6, 0x3e, 0xa7, 0xb1, 0xf4, 0x52, 0x5f,
-	0x4e, 0x31, 0x26, 0xb6, 0x3b, 0x88, 0xe9, 0xd9, 0x5b, 0xa4, 0xbe, 0xf5, 0xe5, 0xd4, 0xf9, 0x5b,
-	0x03, 0xec, 0x32, 0x98, 0xe4, 0x03, 0xe8, 0xa8, 0x63, 0x3d, 0x16, 0xe6, 0x9e, 0x68, 0xab, 0xe5,
-	0xab, 0x50, 0x55, 0x45, 0x72, 0x7c, 0x2c, 0xa8, 0x44, 0xf3, 0x9a, 0x6e, 0xbe, 0x52, 0x99, 0x25,
-	0xd8, 0x37, 0xba, 0x10, 0x56, 0x5c, 0xfc, 0x56, 0x1e, 0x9f, 0x49, 0x36, 0xa3, 0x78, 0x60, 0xd3,
-	0xd5, 0x0b, 0x32, 0x84, 0x16, 0xf5, 0xa4, 0x3f, 0xc1, 0x0c, 0xb7, 0xdd, 0x15, 0xfa, 0xce, 0x9f,
-	0x90, 0x1f, 0xc1, 0x4d, 0x91, 0x64, 0x3c, 0xa0, 0x5e, 0x71, 0x6c, 0x1b, 0xb9, 0x7d, 0x4d, 0xdd,
-	0xd7, 0x87, 0x3b, 0xd0, 0x3c, 0x66, 0xe1, 0xa8, 0x83, 0x8e, 0x59, 0xad, 0x26, 0xe1, 0xab, 0xd0,
-	0x55, 0x4c, 0xf2, 0x53, 0x80, 0x52, 0x53, 0x38, 0xea, 0x5e, 0x22, 0x6a, 0x17, 0x7a, 0x43, 0xb2,
-	0x09, 0x10, 0xb0, 0x74, 0x4a, 0xb9, 0xa7, 0x12, 0xc6, 0xc6, 0xe4, 0xb0, 0x35, 0xe5, 0x0b, 0x7a,
-	0xa1, 0xd8, 0x4c, 0x78, 0x93, 0x6f, 0x58, 0x9a, 0xd2, 0x70, 0x04, 0xe8, 0x61, 0x9b, 0x89, 0xcf,
-	0x35, 0xc1, 0xf9, 0x0a, 0xda, 0xb9, 0x71, 0xf7, 0xc0, 0x3e, 0x4d, 0xa2, 0x6c, 0x56, 0x3a, 0x6d,
-	0xe0, 0x76, 0x35, 0xe1, 0x55, 0x48, 0xee, 0x02, 0xa2, 0x24, 0x1e, 0xd1, 0x40, 0x17, 0xa1, 0x7f,
-	0xd5, 0x01, 0x77, 0xa0, 0x1d, 0x24, 0xc9, 0x09, 0xd3, 0xbe, 0xeb, 0xb8, 0xf9, 0xca, 0xf9, 0x5f,
-	0x03, 0x6e, 0x56, 0x8b, 0x45, 0x1d, 0x81, 0x5a, 0xd0, 0xd3, 0x16, 0xaa, 0x41, 0xb5, 0x87, 0x15,
-	0x6f, 0x37, 0x4c, 0x6f, 0x17, 0x5b, 0x66, 0x49, 0xa8, 0x0f, 0x18, 0xe8, 0x2d, 0x6f, 0x92, 0x90,
-	0xaa, 0x5c, 0xcf, 0x58, 0x88, 0xe1, 0x19, 0xb8, 0xea, 0x53, 0x51, 0x26, 0x2c, 0xcc, 0xc1, 0x47,
-	0x7d, 0xa2, 0x79, 0x1c, 0xf5, 0xb6, 0x75, 0xc0, 0xf5, 0x4a, 0x05, 0x7c, 0xa6, 0xa8, 0x1d, 0x1d,
-	0x45, 0xf5, 0x4d, 0xb6, 0xa0, 0xc7, 0x69, 0x1a, 0xe5, 0xb9, 0x8f, 0xce, 0xb7, 0x5d, 0x93, 0x44,
-	0xee, 0x03, 0x04, 0x49, 0x14, 0xd1, 0x00, 0x05, 0x6c, 0x14, 0x30, 0x28, 0x2a, 0xef, 0xa4, 0x8c,
-	0x3c, 0x41, 0x03, 0x74, 0x75, 0xcb, 0x6d, 0x4b, 0x19, 0x1d, 0xd2, 0x40, 0xdd, 0x23, 0x13, 0x94,
-	0x7b, 0x08, 0x5f, 0x3d, 0xdc, 0xd7, 0x55, 0x04, 0x04, 0xd9, 0x4d, 0x80, 0x09, 0x4f, 0xb2, 0x54,
-	0x73, 0xfb, 0x5b, 0x4d, 0x85, 0xe4, 0x48, 0x41, 0xf6, 0x23, 0xb8, 0x29, 0x2e, 0x66, 0x11, 0x8b,
-	0x4f, 0x3c, 0xe9, 0xf3, 0x09, 0x95, 0xa3, 0x81, 0xae, 0x80, 0x9c, 0xfa, 0x0e, 0x89, 0x4e, 0x0a,
-	0x64, 0x97, 0x53, 0x5f, 0xd2, 0xef, 0xd0, 0xb4, 0xbe, 0x1d, 0x36, 0x90, 0xdb, 0xd0, 0x4e, 0x3c,
-	0x7a, 0x1e, 0x44, 0x79, 0x89, 0xb6, 0x92, 0xbd, 0xf3, 0x20, 0x72, 0x9e, 0xc0, 0xb0, 0x72, 0x62,
-	0x0e, 0xeb, 0xeb, 0xd0, 0xa2, 0x9c, 0x27, 0x05, 0x08, 0xe9, 0x85, 0xf3, 0x3b, 0x20, 0xef, 0xd3,
-	0xf0, 0x87, 0x30, 0xcf, 0xb9, 0x0d, 0xc3, 0x8a, 0x6a, 0x6d, 0x87, 0xf3, 0x2f, 0x0b, 0xc8, 0x4b,
-	0xc4, 0x92, 0xef, 0xd7, 0xc6, 0x55, 0x75, 0xab, 0x16, 0xa3, 0xb1, 0x2a, 0xf4, 0xa5, 0x9f, 0x37,
-	0xc0, 0x3e, 0x13, 0x5a, 0xff, 0x4b, 0x5f, 0xfa, 0x79, 0x23, 0xe2, 0x34, 0xc8, 0xb8, 0xea, 0x89,
-	0x98, 0x84, 0xd8, 0x88, 0xdc, 0x82, 0x44, 0x3e, 0x86, 0x3b, 0x6c, 0x12, 0x27, 0x9c, 0xce, 0xc5,
-	0x3c, 0xed, 0xaa, 0x36, 0x0a, 0xaf, 0x6b, 0x6e, 0xb9, 0x61, 0x0f, 0x3d, 0xf7, 0x04, 0x86, 0x95,
-	0x6b, 0x5c, 0xe9, 0xe6, 0x3f, 0x5b, 0x30, 0x7a, 0x2e, 0x93, 0x19, 0x0b, 0x5c, 0xaa, 0x8c, 0xaf,
-	0x5c, 0xfd, 0x21, 0x0c, 0x14, 0x9a, 0x2f, 0x5e, 0xbf, 0x9f, 0x44, 0xe1, 0xbc, 0x5b, 0xde, 0x05,
-	0x05, 0xe8, 0x9e, 0xe1, 0x85, 0x4e, 0x12, 0x85, 0x98, 0x89, 0x0f, 0x41, 0xa1, 0xae, 0xb1, 0x5f,
-	0xcf, 0x0d, 0xfd, 0x98, 0x9e, 0x55, 0xf6, 0x2b, 0x21, 0xdc, 0xaf, 0xa1, 0xba, 0x13, 0xd3, 0x33,
-	0xb5, 0xdf, 0xb9, 0x07, 0x77, 0x97, 0xd8, 0x96, 0x87, 0xeb, 0xdf, 0x16, 0x0c, 0x9f, 0x0b, 0xc1,
-	0x26, 0xf1, 0x6f, 0x11, 0x76, 0x0a, 0xa3, 0xd7, 0xa1, 0x15, 0x24, 0x59, 0x2c, 0xd1, 0xd8, 0x96,
-	0xab, 0x17, 0x0b, 0x95, 0xd8, 0xa8, 0x55, 0xe2, 0x42, 0x2d, 0x37, 0xeb, 0xb5, 0x6c, 0xd4, 0xea,
-	0x4a, 0xa5, 0x56, 0x1f, 0x40, 0x4f, 0x05, 0xd9, 0x0b, 0x68, 0x2c, 0x29, 0xcf, 0x71, 0x1e, 0x14,
-	0x69, 0x17, 0x29, 0x4a, 0xc0, 0xec, 0x47, 0x1a, 0xea, 0x21, 0x9d, 0x37, 0xa3, 0xff, 0x58, 0xb0,
-	0x5e, 0xbd, 0x4a, 0x1e, 0xb3, 0x4b, 0xfb, 0x92, 0x82, 0x32, 0x1e, 0xe5, 0xf7, 0x50, 0x9f, 0x0a,
-	0x14, 0xd2, 0xec, 0x28, 0x62, 0x81, 0xa7, 0x18, 0xda, 0x7e, 0x5b, 0x53, 0xde, 0xf3, 0x68, 0xee,
-	0x95, 0x15, 0xd3, 0x2b, 0x04, 0x56, 0xfc, 0x4c, 0x4e, 0x8b, 0xde, 0xa4, 0xbe, 0x17, 0x3c, 0xd5,
-	0xbe, 0xce, 0x53, 0x9d, 0xba, 0xa7, 0xca, 0x4c, 0xeb, 0x9a, 0x99, 0xf6, 0x31, 0x0c, 0xf5, 0x70,
-	0x5b, 0x0d, 0xd7, 0x26, 0x40, 0xd9, 0x47, 0xc4, 0xc8, 0xd2, 0x60, 0x56, 0x34, 0x12, 0xe1, 0xfc,
-	0x12, 0xec, 0xd7, 0x89, 0xd6, 0x2b, 0xc8, 0x33, 0xb0, 0xa3, 0x62, 0x81, 0xa2, 0xbd, 0x1d, 0x32,
-	0xaf, 0xf1, 0x42, 0xce, 0x9d, 0x0b, 0x39, 0x9f, 0x41, 0xb7, 0x20, 0x17, 0x3e, 0xb3, 0x2e, 0xf3,
-	0x59, 0x63, 0xc1, 0x67, 0xce, 0x3f, 0x2d, 0x58, 0xaf, 0x9a, 0x9c, 0x87, 0xe5, 0x3d, 0x0c, 0xca,
-	0x23, 0xbc, 0x99, 0x9f, 0xe6, 0xb6, 0x3c, 0x33, 0x6d, 0xa9, 0x6f, 0x2b, 0x0d, 0x14, 0x6f, 0xfc,
-	0x54, 0xe7, 0x72, 0x3f, 0x32, 0x48, 0xe3, 0x77, 0xb0, 0x56, 0x13, 0x59, 0x32, 0xd9, 0xfd, 0xc4,
-	0x9c, 0xec, 0x2a, 0xd3, 0x69, 0xb9, 0xdb, 0x1c, 0xf7, 0x3e, 0x85, 0x0f, 0x34, 0x1c, 0xec, 0x96,
-	0x31, 0x2c, 0x7c, 0x5f, 0x0d, 0xb5, 0xb5, 0x18, 0x6a, 0x67, 0x0c, 0xa3, 0xfa, 0xd6, 0xbc, 0xfc,
-	0x26, 0xb0, 0x76, 0x28, 0x7d, 0xc9, 0x84, 0x64, 0x41, 0xf9, 0xc4, 0x58, 0xc8, 0x0d, 0xeb, 0xba,
-	0x8e, 0x58, 0xaf, 0xc3, 0x55, 0x68, 0x4a, 0x59, 0xe4, 0xaf, 0xfa, 0x54, 0x51, 0x20, 0xe6, 0x49,
-	0x79, 0x0c, 0x7e, 0x80, 0xa3, 0x54, 0x3e, 0xc8, 0x44, 0xfa, 0x91, 0x9e, 0x38, 0x56, 0x70, 0xe2,
-	0xb0, 0x91, 0x82, 0x23, 0x87, 0x6e, 0xca, 0xa1, 0xe6, 0xb6, 0xf4, 0x3c, 0xa2, 0x08, 0xc8, 0xdc,
-	0x04, 0xc0, 0x52, 0xd5, 0x55, 0xd6, 0xd6, 0x7b, 0x15, 0x65, 0x57, 0x11, 0x9c, 0xfb, 0xb0, 0xf1,
-	0x39, 0x95, 0x6a, 0x76, 0xe2, 0xbb, 0x49, 0x7c, 0xcc, 0x26, 0x19, 0xf7, 0x8d, 0x50, 0x38, 0xff,
-	0xb5, 0x60, 0xf3, 0x12, 0x81, 0xfc, 0xc2, 0x23, 0xe8, 0xcc, 0x7c, 0x21, 0x29, 0x2f, 0xaa, 0xa4,
-	0x58, 0x2e, 0xba, 0xa2, 0x71, 0x9d, 0x2b, 0x9a, 0x35, 0x57, 0xdc, 0x86, 0xf6, 0xcc, 0x3f, 0xf7,
-	0x66, 0x47, 0xf9, 0x70, 0xd4, 0x9a, 0xf9, 0xe7, 0x6f, 0x8e, 0x10, 0xd9, 0x18, 0xf7, 0x8e, 0xb2,
-	0xe0, 0x84, 0x4a, 0x51, 0x22, 0x1b, 0xe3, 0x2f, 0x34, 0x45, 0x5d, 0x5a, 0x09, 0x7c, 0x9d, 0xd1,
-	0x8c, 0x8a, 0x1c, 0x2b, 0x54, 0x73, 0xfc, 0x0d, 0x12, 0x70, 0x98, 0xc2, 0xc9, 0x12, 0x51, 0xa2,
-	0xeb, 0xe6, 0x2b, 0x27, 0x83, 0x3b, 0xea, 0x7d, 0x47, 0xe3, 0xfd, 0x84, 0xe3, 0x1b, 0xa2, 0x4c,
-	0xa0, 0x07, 0xd0, 0x0b, 0x22, 0xa6, 0xa0, 0xd2, 0x78, 0xb8, 0x81, 0x26, 0x61, 0x4b, 0x41, 0x2c,
-	0x95, 0x53, 0xaf, 0xf2, 0x56, 0x05, 0x45, 0x7a, 0xab, 0xdf, 0xab, 0x77, 0xa1, 0x2b, 0x58, 0x1c,
-	0x50, 0x2f, 0xd6, 0x0f, 0x84, 0xa6, 0xdb, 0xc1, 0xf5, 0x81, 0x70, 0xfe, 0x64, 0xc1, 0x6d, 0x7c,
-	0xf9, 0xd4, 0x9e, 0x2d, 0x57, 0xf7, 0xf8, 0x5f, 0x03, 0xa1, 0xa7, 0x68, 0x93, 0xb1, 0x27, 0xaf,
-	0xbe, 0x7b, 0xc6, 0x8c, 0xb1, 0xa8, 0xd6, 0x5d, 0xa3, 0x8b, 0x24, 0xc7, 0x57, 0x80, 0x34, 0xd1,
-	0xa5, 0x3d, 0x84, 0x96, 0x14, 0x1e, 0x42, 0x99, 0xb2, 0x73, 0x45, 0x8a, 0x03, 0x41, 0x9e, 0x02,
-	0x49, 0x7d, 0x2e, 0x99, 0x92, 0x56, 0xf3, 0xb3, 0x37, 0xf5, 0xc5, 0x14, 0x0f, 0x6b, 0xb9, 0xab,
-	0x25, 0xe7, 0x0b, 0x7a, 0xf1, 0x2b, 0x5f, 0x4c, 0x15, 0x80, 0xe3, 0x80, 0xd1, 0xc4, 0x39, 0x1e,
-	0xbf, 0x77, 0xfe, 0xda, 0x85, 0xfe, 0x21, 0xf5, 0xcf, 0x28, 0x0d, 0x31, 0x9d, 0xc8, 0xa4, 0x80,
-	0xb1, 0xea, 0x6f, 0x05, 0xf2, 0x68, 0x11, 0xaf, 0x96, 0xfe, 0xc7, 0x18, 0x7f, 0x74, 0x9d, 0x58,
-	0x8e, 0x08, 0x37, 0xc8, 0x01, 0xf4, 0x8c, 0x77, 0x3b, 0xd9, 0x30, 0x36, 0xd6, 0x7e, 0x47, 0x8c,
-	0x37, 0x2f, 0xe1, 0x16, 0xda, 0x9e, 0x59, 0xe4, 0x35, 0xf4, 0x8c, 0x81, 0xd1, 0xd4, 0x57, 0x9f,
-	0x5c, 0x4d, 0x7d, 0x4b, 0xa6, 0x4c, 0xe7, 0x86, 0xd2, 0x66, 0x8c, 0x7d, 0xa6, 0xb6, 0xfa, 0xa0,
-	0x69, 0x6a, 0x5b, 0x36, 0x2b, 0xa2, 0x36, 0x63, 0xca, 0x32, 0xb5, 0xd5, 0x67, 0x48, 0x53, 0xdb,
-	0x92, 0xd1, 0xcc, 0xb9, 0x41, 0xfe, 0x00, 0x6b, 0xb5, 0x49, 0x87, 0x38, 0xf3, 0x5d, 0x97, 0x8d,
-	0x68, 0xe3, 0x87, 0x57, 0xca, 0x94, 0xfa, 0xbf, 0x84, 0xbe, 0x39, 0x60, 0x10, 0xc3, 0xa0, 0x25,
-	0x33, 0xd4, 0xf8, 0xfe, 0x65, 0x6c, 0x53, 0xa1, 0xd9, 0xe3, 0x4c, 0x85, 0x4b, 0xba, 0xbc, 0xa9,
-	0x70, 0x59, 0x6b, 0x74, 0x6e, 0x90, 0xdf, 0xc3, 0xea, 0x62, 0xaf, 0x21, 0x1f, 0x2e, 0xba, 0xad,
-	0xd6, 0xc2, 0xc6, 0xce, 0x55, 0x22, 0xa5, 0xf2, 0x57, 0x00, 0xf3, 0x16, 0x42, 0x8c, 0x9a, 0xad,
-	0xb5, 0xb0, 0xf1, 0xc6, 0x72, 0x66, 0xa9, 0xea, 0x8f, 0x70, 0x7b, 0x29, 0x4e, 0x13, 0xa3, 0x4c,
-	0xae, 0x42, 0xfa, 0xf1, 0x8f, 0xaf, 0x95, 0x2b, 0xcf, 0xfa, 0x0a, 0x6e, 0x2d, 0xe0, 0x24, 0xd9,
-	0xaa, 0x56, 0x4d, 0x1d, 0x42, 0xc7, 0x0f, 0xcc, 0x9f, 0x4f, 0x4b, 0xc0, 0x4e, 0x55, 0xd6, 0x8b,
-	0xfb, 0xb0, 0x2a, 0x34, 0x44, 0x1c, 0x8b, 0x6d, 0x0d, 0xaf, 0x2f, 0x00, 0x6d, 0x79, 0xcb, 0x13,
-	0x99, 0x1c, 0xb5, 0xf1, 0x5f, 0xe7, 0xcf, 0xfe, 0x1f, 0x00, 0x00, 0xff, 0xff, 0x4d, 0x32, 0x75,
-	0x14, 0xfa, 0x14, 0x00, 0x00,
+	0xd7, 0xa9, 0x0b, 0x1b, 0xaa, 0xa1, 0xa6, 0x40, 0xd2, 0xb4, 0x0f, 0xb6, 0x2c, 0xa5, 0x6e, 0x6c,
+	0xc5, 0xa5, 0xec, 0x22, 0x45, 0x81, 0x12, 0x14, 0xb9, 0xba, 0xdb, 0x8a, 0x47, 0x32, 0xbb, 0x4b,
+	0xfd, 0xc9, 0x53, 0xfb, 0x35, 0x0a, 0xf4, 0xa1, 0xdf, 0xa1, 0x8f, 0x45, 0x5f, 0x8a, 0x02, 0xfd,
+	0x1c, 0x7d, 0xec, 0xa7, 0x28, 0x76, 0x96, 0xe4, 0x2d, 0x8f, 0x27, 0x29, 0x41, 0x91, 0xb7, 0xdd,
+	0x99, 0xd9, 0xd9, 0xd9, 0xf9, 0xf3, 0x9b, 0x21, 0xa1, 0x77, 0xcc, 0x22, 0xca, 0xb7, 0x53, 0x9e,
+	0xc8, 0x84, 0x74, 0x71, 0xe3, 0xa5, 0x47, 0xce, 0x97, 0x70, 0xef, 0x75, 0x92, 0x9c, 0x64, 0xe9,
+	0x4b, 0xc6, 0x69, 0x20, 0x13, 0x7e, 0xb1, 0x17, 0x4b, 0x7e, 0xe1, 0xd2, 0xaf, 0x33, 0x2a, 0x24,
+	0xd9, 0x00, 0x3b, 0x2c, 0x18, 0x23, 0x6b, 0xcb, 0x7a, 0x6c, 0xbb, 0x73, 0x02, 0x21, 0xb0, 0x12,
+	0xfb, 0x33, 0x3a, 0x6a, 0x20, 0x03, 0xd7, 0xce, 0x1e, 0x6c, 0x2c, 0x57, 0x28, 0xd2, 0x24, 0x16,
+	0x94, 0x3c, 0x82, 0x16, 0x55, 0x04, 0xd4, 0xd6, 0xdb, 0xb9, 0xb5, 0x5d, 0x98, 0xb2, 0xad, 0xe5,
+	0x34, 0xd7, 0xf9, 0x87, 0x05, 0xe4, 0x35, 0x13, 0x52, 0x11, 0x19, 0x15, 0xdf, 0xce, 0x9e, 0x3b,
+	0xd0, 0x4e, 0x39, 0x3d, 0x66, 0xe7, 0xb9, 0x45, 0xf9, 0x8e, 0x3c, 0x85, 0x35, 0x21, 0x7d, 0x2e,
+	0xf7, 0x79, 0x32, 0xdb, 0x67, 0x11, 0x3d, 0x50, 0x46, 0x37, 0x51, 0xa4, 0xce, 0x20, 0xdb, 0x40,
+	0x58, 0x1c, 0x44, 0x99, 0x60, 0xa7, 0xf4, 0xb0, 0xe0, 0x8e, 0x56, 0xb6, 0xac, 0xc7, 0x5d, 0x77,
+	0x09, 0x87, 0xac, 0x43, 0x2b, 0x62, 0x33, 0x26, 0x47, 0xad, 0x2d, 0xeb, 0xf1, 0xc0, 0xd5, 0x1b,
+	0xe7, 0xe7, 0x30, 0xac, 0xd8, 0xff, 0xdd, 0x9e, 0xff, 0x97, 0x06, 0xb4, 0x90, 0x50, 0xfa, 0xd8,
+	0x9a, 0xfb, 0x98, 0x7c, 0x08, 0x7d, 0x26, 0xbc, 0xb9, 0x23, 0x1a, 0x68, 0x5b, 0x8f, 0x89, 0xd2,
+	0xe7, 0xe4, 0x09, 0xb4, 0x83, 0x69, 0x16, 0x9f, 0x88, 0x51, 0x73, 0xab, 0xf9, 0xb8, 0xb7, 0x33,
+	0x9c, 0x5f, 0xa4, 0x1e, 0xba, 0xab, 0x78, 0x6e, 0x2e, 0x42, 0x3e, 0x01, 0xf0, 0xa5, 0xe4, 0xec,
+	0x28, 0x93, 0x54, 0xe0, 0x4b, 0x7b, 0x3b, 0x23, 0xe3, 0x40, 0x26, 0xe8, 0xf3, 0x92, 0xef, 0x1a,
+	0xb2, 0xe4, 0x53, 0xe8, 0xd2, 0x73, 0x49, 0xe3, 0x90, 0x86, 0xa3, 0x16, 0x5e, 0xb4, 0xb9, 0xf0,
+	0xa2, 0xed, 0xbd, 0x9c, 0xaf, 0xdf, 0x57, 0x8a, 0x8f, 0x3f, 0x83, 0x41, 0x85, 0x45, 0x56, 0xa1,
+	0x79, 0x42, 0x8b, 0xa8, 0xaa, 0xa5, 0xf2, 0xec, 0xa9, 0x1f, 0x65, 0x3a, 0xc1, 0xfa, 0xae, 0xde,
+	0xfc, 0xac, 0xf1, 0x89, 0xe5, 0xbc, 0x04, 0x7b, 0x3f, 0x8b, 0xa2, 0xf2, 0x60, 0xc8, 0x78, 0x71,
+	0x30, 0x64, 0x7c, 0xee, 0xe5, 0xc6, 0x95, 0x5e, 0xfe, 0xbb, 0x05, 0x6b, 0x7b, 0xa7, 0x34, 0x96,
+	0x07, 0x89, 0x64, 0xc7, 0x2c, 0xf0, 0x25, 0x4b, 0x62, 0xf2, 0x14, 0xec, 0x24, 0x0a, 0xbd, 0x2b,
+	0xc3, 0xd4, 0x4d, 0xa2, 0xdc, 0xea, 0xa7, 0x60, 0xc7, 0xf4, 0xcc, 0xbb, 0xf2, 0xba, 0x6e, 0x4c,
+	0xcf, 0xb4, 0xf4, 0x43, 0x18, 0x84, 0x34, 0xa2, 0x92, 0x7a, 0x65, 0x74, 0x54, 0xe8, 0xfa, 0x9a,
+	0xb8, 0xab, 0xc3, 0xf1, 0x11, 0xdc, 0x52, 0x2a, 0x53, 0x9f, 0xd3, 0x58, 0x7a, 0xa9, 0x2f, 0xa7,
+	0x18, 0x13, 0xdb, 0x1d, 0xc4, 0xf4, 0xec, 0x2d, 0x52, 0xdf, 0xfa, 0x72, 0xea, 0xfc, 0xad, 0x01,
+	0x76, 0x19, 0x4c, 0xf2, 0x01, 0x74, 0xd4, 0xb5, 0x1e, 0x0b, 0x73, 0x4f, 0xb4, 0xd5, 0xf6, 0x55,
+	0xa8, 0xaa, 0x22, 0x39, 0x3e, 0x16, 0x54, 0xa2, 0x79, 0x4d, 0x37, 0xdf, 0xa9, 0xcc, 0x12, 0xec,
+	0x1b, 0x5d, 0x08, 0x2b, 0x2e, 0xae, 0x95, 0xc7, 0x67, 0x92, 0xcd, 0x28, 0x5e, 0xd8, 0x74, 0xf5,
+	0x86, 0x0c, 0xa1, 0x45, 0x3d, 0xe9, 0x4f, 0x30, 0xc3, 0x6d, 0x77, 0x85, 0xbe, 0xf3, 0x27, 0xe4,
+	0x07, 0x70, 0x53, 0x24, 0x19, 0x0f, 0xa8, 0x57, 0x5c, 0xdb, 0x46, 0x6e, 0x5f, 0x53, 0xf7, 0xf5,
+	0xe5, 0x0e, 0x34, 0x8f, 0x59, 0x38, 0xea, 0xa0, 0x63, 0x56, 0xab, 0x49, 0xf8, 0x2a, 0x74, 0x15,
+	0x93, 0xfc, 0x18, 0xa0, 0xd4, 0x14, 0x8e, 0xba, 0x97, 0x88, 0xda, 0x85, 0xde, 0x90, 0x6c, 0x02,
+	0x04, 0x2c, 0x9d, 0x52, 0xee, 0xa9, 0x84, 0xb1, 0x31, 0x39, 0x6c, 0x4d, 0xf9, 0x82, 0x5e, 0x28,
+	0x36, 0x13, 0xde, 0xe4, 0x1b, 0x96, 0xa6, 0x34, 0x1c, 0x01, 0x7a, 0xd8, 0x66, 0xe2, 0x73, 0x4d,
+	0x70, 0xbe, 0x82, 0x76, 0x6e, 0xdc, 0x3d, 0xb0, 0x4f, 0x93, 0x28, 0x9b, 0x95, 0x4e, 0x1b, 0xb8,
+	0x5d, 0x4d, 0x78, 0x15, 0x92, 0xbb, 0x80, 0x28, 0x89, 0x57, 0x34, 0xd0, 0x45, 0xe8, 0x5f, 0x75,
+	0xc1, 0x1d, 0x68, 0x07, 0x49, 0x72, 0xc2, 0xb4, 0xef, 0x3a, 0x6e, 0xbe, 0x73, 0xfe, 0xd8, 0x84,
+	0x9b, 0xd5, 0x62, 0x51, 0x57, 0xa0, 0x16, 0xf4, 0xb4, 0x85, 0x6a, 0x50, 0xed, 0x61, 0xc5, 0xdb,
+	0x0d, 0xd3, 0xdb, 0xc5, 0x91, 0x59, 0x12, 0xea, 0x0b, 0x06, 0xfa, 0xc8, 0x9b, 0x24, 0xa4, 0x2a,
+	0xd7, 0x33, 0x16, 0x62, 0x78, 0x06, 0xae, 0x5a, 0x2a, 0xca, 0x84, 0x85, 0x39, 0xf8, 0xa8, 0x25,
+	0x9a, 0xc7, 0x51, 0x6f, 0x5b, 0x07, 0x5c, 0xef, 0x54, 0xc0, 0x67, 0x8a, 0xda, 0xd1, 0x51, 0x54,
+	0x6b, 0xb2, 0x05, 0x3d, 0x4e, 0xd3, 0x28, 0xcf, 0x7d, 0x74, 0xbe, 0xed, 0x9a, 0x24, 0x72, 0x1f,
+	0x20, 0x48, 0xa2, 0x88, 0x06, 0x28, 0x60, 0xa3, 0x80, 0x41, 0x51, 0x79, 0x27, 0x65, 0xe4, 0x09,
+	0x1a, 0xa0, 0xab, 0x5b, 0x6e, 0x5b, 0xca, 0xe8, 0x90, 0x06, 0xea, 0x1d, 0x99, 0xa0, 0xdc, 0x43,
+	0xf8, 0xea, 0xe1, 0xb9, 0xae, 0x22, 0x20, 0xc8, 0x6e, 0x02, 0x4c, 0x78, 0x92, 0xa5, 0x9a, 0xdb,
+	0xdf, 0x6a, 0x2a, 0x24, 0x47, 0x0a, 0xb2, 0x1f, 0xc1, 0x4d, 0x71, 0x31, 0x8b, 0x58, 0x7c, 0xe2,
+	0x49, 0x9f, 0x4f, 0xa8, 0x1c, 0x0d, 0x74, 0x05, 0xe4, 0xd4, 0x77, 0x48, 0x54, 0x6f, 0x9f, 0x85,
+	0x3f, 0x1d, 0xdd, 0xc4, 0x0c, 0x50, 0x4b, 0x27, 0x05, 0xb2, 0xcb, 0xa9, 0x2f, 0xe9, 0x77, 0x68,
+	0x63, 0xdf, 0x0e, 0x2d, 0xc8, 0x6d, 0x68, 0x27, 0x1e, 0x3d, 0x0f, 0xa2, 0xbc, 0x68, 0x5b, 0xc9,
+	0xde, 0x79, 0x10, 0x39, 0x4f, 0x60, 0x58, 0xb9, 0x31, 0x07, 0xfa, 0x75, 0x68, 0x51, 0xce, 0x93,
+	0x02, 0x96, 0xf4, 0xc6, 0xf9, 0x2d, 0x90, 0xf7, 0x69, 0xf8, 0x7d, 0x98, 0xe7, 0xdc, 0x86, 0x61,
+	0x45, 0xb5, 0xb6, 0xc3, 0xf9, 0x97, 0x05, 0xe4, 0x25, 0xa2, 0xcb, 0xff, 0xd7, 0xd8, 0x55, 0xbd,
+	0xab, 0xa6, 0xa3, 0xd1, 0x2b, 0xf4, 0xa5, 0x9f, 0xb7, 0xc4, 0x3e, 0x13, 0x5a, 0xff, 0x4b, 0x5f,
+	0xfa, 0x79, 0x6b, 0xe2, 0x34, 0xc8, 0xb8, 0xea, 0x92, 0x98, 0x96, 0xd8, 0x9a, 0xdc, 0x82, 0x44,
+	0x3e, 0x86, 0x3b, 0x6c, 0x12, 0x27, 0x9c, 0xce, 0xc5, 0x3c, 0xed, 0xaa, 0x36, 0x0a, 0xaf, 0x6b,
+	0x6e, 0x79, 0x60, 0x0f, 0x3d, 0xf7, 0x04, 0x86, 0x95, 0x67, 0x5c, 0xe9, 0xe6, 0x3f, 0x5b, 0x30,
+	0x7a, 0x2e, 0x93, 0x19, 0x0b, 0x5c, 0xaa, 0x8c, 0xaf, 0x3c, 0xfd, 0x21, 0x0c, 0x14, 0xbe, 0x2f,
+	0x3e, 0xbf, 0x9f, 0x44, 0xe1, 0xbc, 0x7f, 0xde, 0x05, 0x05, 0xf1, 0x9e, 0xe1, 0x85, 0x4e, 0x12,
+	0x85, 0x98, 0x9b, 0x0f, 0x41, 0xe1, 0xb0, 0x71, 0x5e, 0x4f, 0x12, 0xfd, 0x98, 0x9e, 0x55, 0xce,
+	0x2b, 0x21, 0x3c, 0xaf, 0xc1, 0xbb, 0x13, 0xd3, 0x33, 0x75, 0xde, 0xb9, 0x07, 0x77, 0x97, 0xd8,
+	0x96, 0x87, 0xeb, 0xdf, 0x16, 0x0c, 0x9f, 0x0b, 0xc1, 0x26, 0xf1, 0x6f, 0x10, 0x88, 0x0a, 0xa3,
+	0xd7, 0xa1, 0x15, 0x24, 0x59, 0x2c, 0xd1, 0xd8, 0x96, 0xab, 0x37, 0x0b, 0xb5, 0xd9, 0xa8, 0xd5,
+	0xe6, 0x42, 0x75, 0x37, 0xeb, 0xd5, 0x6d, 0x54, 0xef, 0x4a, 0xa5, 0x7a, 0x1f, 0x40, 0x4f, 0x05,
+	0xd9, 0x0b, 0x68, 0x2c, 0x29, 0xcf, 0x91, 0x1f, 0x14, 0x69, 0x17, 0x29, 0x4a, 0xc0, 0xec, 0x50,
+	0x1a, 0xfc, 0x21, 0x9d, 0xb7, 0xa7, 0xff, 0x58, 0xb0, 0x5e, 0x7d, 0x4a, 0x1e, 0xb3, 0x4b, 0x3b,
+	0x95, 0x02, 0x37, 0x1e, 0xe5, 0xef, 0x50, 0x4b, 0x05, 0x13, 0x69, 0x76, 0x14, 0xb1, 0xc0, 0x53,
+	0x0c, 0x6d, 0xbf, 0xad, 0x29, 0xef, 0x79, 0x34, 0xf7, 0xca, 0x8a, 0xe9, 0x15, 0x02, 0x2b, 0x7e,
+	0x26, 0xa7, 0x45, 0xb7, 0x52, 0xeb, 0x05, 0x4f, 0xb5, 0xaf, 0xf3, 0x54, 0xa7, 0xee, 0xa9, 0x32,
+	0xd3, 0xba, 0x66, 0xa6, 0x7d, 0x0c, 0x43, 0x3d, 0xee, 0x56, 0xc3, 0xb5, 0x09, 0x50, 0x76, 0x16,
+	0x31, 0xb2, 0x34, 0xbc, 0x15, 0xad, 0x45, 0x38, 0xbf, 0x00, 0xfb, 0x75, 0xa2, 0xf5, 0x0a, 0xf2,
+	0x0c, 0xec, 0xa8, 0xd8, 0xa0, 0x68, 0x6f, 0x87, 0xcc, 0x6b, 0xbc, 0x90, 0x73, 0xe7, 0x42, 0xce,
+	0x67, 0xd0, 0x2d, 0xc8, 0x85, 0xcf, 0xac, 0xcb, 0x7c, 0xd6, 0x58, 0xf0, 0x99, 0xf3, 0x4f, 0x0b,
+	0xd6, 0xab, 0x26, 0xe7, 0x61, 0x79, 0x0f, 0x83, 0xf2, 0x0a, 0x6f, 0xe6, 0xa7, 0xb9, 0x2d, 0xcf,
+	0x4c, 0x5b, 0xea, 0xc7, 0x4a, 0x03, 0xc5, 0x1b, 0x3f, 0xd5, 0xb9, 0xdc, 0x8f, 0x0c, 0xd2, 0xf8,
+	0x1d, 0xac, 0xd5, 0x44, 0x96, 0xcc, 0x7a, 0x3f, 0x32, 0x67, 0xbd, 0xca, 0xbc, 0x5a, 0x9e, 0x36,
+	0x07, 0xc0, 0x4f, 0xe1, 0x03, 0x0d, 0x07, 0xbb, 0x65, 0x0c, 0x0b, 0xdf, 0x57, 0x43, 0x6d, 0x2d,
+	0x86, 0xda, 0x19, 0xc3, 0xa8, 0x7e, 0x34, 0x2f, 0xbf, 0x09, 0xac, 0x1d, 0x4a, 0x5f, 0x32, 0x21,
+	0x59, 0x50, 0x7e, 0x74, 0x2c, 0xe4, 0x86, 0x75, 0x5d, 0x8f, 0xac, 0xd7, 0xe1, 0x2a, 0x34, 0xa5,
+	0x2c, 0xf2, 0x57, 0x2d, 0x55, 0x14, 0x88, 0x79, 0x53, 0x1e, 0x83, 0xef, 0xe1, 0x2a, 0x95, 0x0f,
+	0x32, 0x91, 0x7e, 0xa4, 0x67, 0x90, 0x15, 0x9c, 0x41, 0x6c, 0xa4, 0xe0, 0x10, 0xa2, 0xdb, 0x74,
+	0xa8, 0xb9, 0x2d, 0x3d, 0xa1, 0x28, 0x02, 0x32, 0x37, 0x01, 0xb0, 0x54, 0x75, 0x95, 0xb5, 0xf5,
+	0x59, 0x45, 0xd9, 0x55, 0x04, 0xe7, 0x3e, 0x6c, 0x7c, 0x4e, 0xa5, 0x9a, 0xa6, 0xf8, 0x6e, 0x12,
+	0x1f, 0xb3, 0x49, 0xc6, 0x7d, 0x23, 0x14, 0xce, 0x7f, 0x2d, 0xd8, 0xbc, 0x44, 0x20, 0x7f, 0xf0,
+	0x08, 0x3a, 0x33, 0x5f, 0x48, 0xca, 0x8b, 0x2a, 0x29, 0xb6, 0x8b, 0xae, 0x68, 0x5c, 0xe7, 0x8a,
+	0x66, 0xcd, 0x15, 0xb7, 0xa1, 0x3d, 0xf3, 0xcf, 0xbd, 0xd9, 0x51, 0x3e, 0x2e, 0xb5, 0x66, 0xfe,
+	0xf9, 0x9b, 0x23, 0x44, 0x36, 0xc6, 0xbd, 0xa3, 0x2c, 0x38, 0xa1, 0x52, 0x94, 0xc8, 0xc6, 0xf8,
+	0x0b, 0x4d, 0x51, 0x8f, 0x56, 0x02, 0x5f, 0x67, 0x34, 0xa3, 0x22, 0xc7, 0x0a, 0xd5, 0x1c, 0x7f,
+	0x8d, 0x04, 0x1c, 0xaf, 0x70, 0xd6, 0x44, 0x94, 0xe8, 0xba, 0xf9, 0xce, 0xc9, 0xe0, 0x8e, 0xfa,
+	0xe2, 0xa3, 0xf1, 0x7e, 0xc2, 0xf1, 0xab, 0xa2, 0x4c, 0xa0, 0x07, 0xd0, 0x0b, 0x22, 0xa6, 0xa0,
+	0xd2, 0xf8, 0x94, 0x03, 0x4d, 0xc2, 0x96, 0x82, 0x58, 0x2a, 0xa7, 0x5e, 0xe5, 0xeb, 0x15, 0x14,
+	0xe9, 0xad, 0xfe, 0x82, 0xbd, 0x0b, 0x5d, 0xc1, 0xe2, 0x80, 0x7a, 0xb1, 0xfe, 0x64, 0x68, 0xba,
+	0x1d, 0xdc, 0x1f, 0x08, 0xe7, 0x4f, 0x16, 0xdc, 0xc6, 0x6f, 0xa1, 0xda, 0x87, 0xcc, 0xd5, 0x3d,
+	0xfe, 0x57, 0x40, 0xe8, 0x29, 0xda, 0x64, 0x9c, 0xc9, 0xab, 0xef, 0x9e, 0x31, 0x63, 0x2c, 0xaa,
+	0x75, 0xd7, 0xe8, 0x22, 0xc9, 0xf1, 0x15, 0x20, 0x4d, 0x74, 0x69, 0x0f, 0xa1, 0x25, 0x85, 0x87,
+	0x50, 0xa6, 0xec, 0x5c, 0x91, 0xe2, 0x40, 0x90, 0xa7, 0x40, 0x52, 0x9f, 0x4b, 0xa6, 0xa4, 0xd5,
+	0x44, 0xed, 0x4d, 0x7d, 0x31, 0xc5, 0xcb, 0x5a, 0xee, 0x6a, 0xc9, 0xf9, 0x82, 0x5e, 0xfc, 0xd2,
+	0x17, 0x53, 0x05, 0xe0, 0x38, 0x60, 0x34, 0x71, 0xae, 0xc3, 0xf5, 0xce, 0x5f, 0xbb, 0xd0, 0x3f,
+	0xa4, 0xfe, 0x19, 0xa5, 0x21, 0xa6, 0x13, 0x99, 0x14, 0x30, 0x56, 0xfd, 0xd1, 0x40, 0x1e, 0x2d,
+	0xe2, 0xd5, 0xd2, 0x3f, 0x1b, 0xe3, 0x8f, 0xae, 0x13, 0xcb, 0x11, 0xe1, 0x06, 0x39, 0x80, 0x9e,
+	0xf1, 0x25, 0x4f, 0x36, 0x8c, 0x83, 0xb5, 0x1f, 0x14, 0xe3, 0xcd, 0x4b, 0xb8, 0x85, 0xb6, 0x67,
+	0x16, 0x79, 0x0d, 0x3d, 0x63, 0x60, 0x34, 0xf5, 0xd5, 0x27, 0x57, 0x53, 0xdf, 0x92, 0x29, 0xd3,
+	0xb9, 0xa1, 0xb4, 0x19, 0x63, 0x9f, 0xa9, 0xad, 0x3e, 0x68, 0x9a, 0xda, 0x96, 0xcd, 0x8a, 0xa8,
+	0xcd, 0x98, 0xb2, 0x4c, 0x6d, 0xf5, 0x19, 0xd2, 0xd4, 0xb6, 0x64, 0x34, 0x73, 0x6e, 0x90, 0xdf,
+	0xc3, 0x5a, 0x6d, 0xd2, 0x21, 0xce, 0xfc, 0xd4, 0x65, 0x23, 0xda, 0xf8, 0xe1, 0x95, 0x32, 0xa5,
+	0xfe, 0x2f, 0xa1, 0x6f, 0x0e, 0x18, 0xc4, 0x30, 0x68, 0xc9, 0x0c, 0x35, 0xbe, 0x7f, 0x19, 0xdb,
+	0x54, 0x68, 0xf6, 0x38, 0x53, 0xe1, 0x92, 0x2e, 0x6f, 0x2a, 0x5c, 0xd6, 0x1a, 0x9d, 0x1b, 0xe4,
+	0x77, 0xb0, 0xba, 0xd8, 0x6b, 0xc8, 0x87, 0x8b, 0x6e, 0xab, 0xb5, 0xb0, 0xb1, 0x73, 0x95, 0x48,
+	0xa9, 0xfc, 0x15, 0xc0, 0xbc, 0x85, 0x10, 0xa3, 0x66, 0x6b, 0x2d, 0x6c, 0xbc, 0xb1, 0x9c, 0x59,
+	0xaa, 0xfa, 0x03, 0xdc, 0x5e, 0x8a, 0xd3, 0xc4, 0x28, 0x93, 0xab, 0x90, 0x7e, 0xfc, 0xc3, 0x6b,
+	0xe5, 0xca, 0xbb, 0xbe, 0x82, 0x5b, 0x0b, 0x38, 0x49, 0xb6, 0xaa, 0x55, 0x53, 0x87, 0xd0, 0xf1,
+	0x03, 0xf3, 0x77, 0xd4, 0x12, 0xb0, 0x53, 0x95, 0xf5, 0xe2, 0x3e, 0xac, 0x0a, 0x0d, 0x11, 0xc7,
+	0x62, 0x5b, 0xc3, 0xeb, 0x0b, 0x40, 0x5b, 0xde, 0xf2, 0x44, 0x26, 0x47, 0x6d, 0xfc, 0xfb, 0xf9,
+	0x93, 0xff, 0x05, 0x00, 0x00, 0xff, 0xff, 0x24, 0x99, 0xf0, 0xba, 0x0c, 0x15, 0x00, 0x00,
 }
diff --git a/weed/replication/sink/filersink/filer_sink.go b/weed/replication/sink/filersink/filer_sink.go
index 5f055f9d1..fa9cc0f05 100644
--- a/weed/replication/sink/filersink/filer_sink.go
+++ b/weed/replication/sink/filersink/filer_sink.go
@@ -90,7 +90,7 @@ func (fs *FilerSink) CreateEntry(key string, entry *filer_pb.Entry) error {
 		}
 		glog.V(1).Infof("lookup: %v", lookupRequest)
 		if resp, err := filer_pb.LookupEntry(client, lookupRequest); err == nil {
-			if filer2.ETag(resp.Entry.Chunks) == filer2.ETag(entry.Chunks) {
+			if filer2.ETag(resp.Entry) == filer2.ETag(entry) {
 				glog.V(0).Infof("already replicated %s", key)
 				return nil
 			}
@@ -160,7 +160,7 @@ func (fs *FilerSink) UpdateEntry(key string, oldEntry *filer_pb.Entry, newParent
 		// skip if already changed
 		// this usually happens when the messages are not ordered
 		glog.V(0).Infof("late updates %s", key)
-	} else if filer2.ETag(newEntry.Chunks) == filer2.ETag(existingEntry.Chunks) {
+	} else if filer2.ETag(newEntry) == filer2.ETag(existingEntry) {
 		// skip if no change
 		// this usually happens when retrying the replication
 		glog.V(0).Infof("already replicated %s", key)
diff --git a/weed/s3api/filer_multipart.go b/weed/s3api/filer_multipart.go
index e81461dd2..85ed99854 100644
--- a/weed/s3api/filer_multipart.go
+++ b/weed/s3api/filer_multipart.go
@@ -107,7 +107,7 @@ func (s3a *S3ApiServer) completeMultipartUpload(input *s3.CompleteMultipartUploa
 		CompleteMultipartUploadOutput: s3.CompleteMultipartUploadOutput{
 			Location: aws.String(fmt.Sprintf("http://%s%s/%s", s3a.option.Filer, dirName, entryName)),
 			Bucket:   input.Bucket,
-			ETag:     aws.String("\"" + filer2.ETag(finalParts) + "\""),
+			ETag:     aws.String("\"" + filer2.ETagChunks(finalParts) + "\""),
 			Key:      objectKey(input.Key),
 		},
 	}
@@ -208,7 +208,7 @@ func (s3a *S3ApiServer) listObjectParts(input *s3.ListPartsInput) (output *ListP
 				PartNumber:   aws.Int64(int64(partNumber)),
 				LastModified: aws.Time(time.Unix(entry.Attributes.Mtime, 0)),
 				Size:         aws.Int64(int64(filer2.TotalSize(entry.Chunks))),
-				ETag:         aws.String("\"" + filer2.ETag(entry.Chunks) + "\""),
+				ETag:         aws.String("\"" + filer2.ETag(entry) + "\""),
 			})
 		}
 	}
diff --git a/weed/s3api/s3api_objects_list_handlers.go b/weed/s3api/s3api_objects_list_handlers.go
index 4469c03a6..086b9acd3 100644
--- a/weed/s3api/s3api_objects_list_handlers.go
+++ b/weed/s3api/s3api_objects_list_handlers.go
@@ -139,7 +139,7 @@ func (s3a *S3ApiServer) listFilerEntries(bucket, originalPrefix string, maxKeys
 				contents = append(contents, ListEntry{
 					Key:          fmt.Sprintf("%s%s", dir, entry.Name),
 					LastModified: time.Unix(entry.Attributes.Mtime, 0),
-					ETag:         "\"" + filer2.ETag(entry.Chunks) + "\"",
+					ETag:         "\"" + filer2.ETag(entry) + "\"",
 					Size:         int64(filer2.TotalSize(entry.Chunks)),
 					Owner: CanonicalUser{
 						ID:          fmt.Sprintf("%x", entry.Attributes.Uid),
diff --git a/weed/server/filer_server_handlers_read.go b/weed/server/filer_server_handlers_read.go
index b59780632..96fc38bf5 100644
--- a/weed/server/filer_server_handlers_read.go
+++ b/weed/server/filer_server_handlers_read.go
@@ -8,6 +8,7 @@ import (
 	"path/filepath"
 	"strconv"
 	"strings"
+	"time"
 
 	"github.com/chrislusf/seaweedfs/weed/filer2"
 	"github.com/chrislusf/seaweedfs/weed/glog"
@@ -78,8 +79,26 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request,
 		w.Header().Set("Content-Type", mimeType)
 	}
 
+	// if modified since
+	if !entry.Attr.Mtime.IsZero() {
+		w.Header().Set("Last-Modified", entry.Attr.Mtime.UTC().Format(http.TimeFormat))
+		if r.Header.Get("If-Modified-Since") != "" {
+			if t, parseError := time.Parse(http.TimeFormat, r.Header.Get("If-Modified-Since")); parseError == nil {
+				if t.After(entry.Attr.Mtime) {
+					w.WriteHeader(http.StatusNotModified)
+					return
+				}
+			}
+		}
+	}
+
 	// set etag
-	setEtag(w, filer2.ETag(entry.Chunks))
+	etag := filer2.ETagEntry(entry)
+	if inm := r.Header.Get("If-None-Match"); inm == "\""+etag+"\"" {
+		w.WriteHeader(http.StatusNotModified)
+		return
+	}
+	setEtag(w, etag)
 
 	if r.Method == "HEAD" {
 		w.Header().Set("Content-Length", strconv.FormatInt(int64(filer2.TotalSize(entry.Chunks)), 10))
diff --git a/weed/server/filer_server_handlers_write.go b/weed/server/filer_server_handlers_write.go
index c3cb48ce7..33e6ee043 100644
--- a/weed/server/filer_server_handlers_write.go
+++ b/weed/server/filer_server_handlers_write.go
@@ -122,12 +122,12 @@ func (fs *FilerServer) PostHandler(w http.ResponseWriter, r *http.Request) {
 	glog.V(4).Infof("write %s to %v", r.URL.Path, urlLocation)
 
 	u, _ := url.Parse(urlLocation)
-	ret, err := fs.uploadToVolumeServer(r, u, auth, w, fileId)
+	ret, md5value, err := fs.uploadToVolumeServer(r, u, auth, w, fileId)
 	if err != nil {
 		return
 	}
 
-	if err = fs.updateFilerStore(ctx, r, w, replication, collection, ret, fileId, ttlSeconds); err != nil {
+	if err = fs.updateFilerStore(ctx, r, w, replication, collection, ret, md5value, fileId, ttlSeconds); err != nil {
 		return
 	}
 
@@ -144,8 +144,8 @@ func (fs *FilerServer) PostHandler(w http.ResponseWriter, r *http.Request) {
 }
 
 // update metadata in filer store
-func (fs *FilerServer) updateFilerStore(ctx context.Context, r *http.Request, w http.ResponseWriter,
-	replication string, collection string, ret *operation.UploadResult, fileId string, ttlSeconds int32) (err error) {
+func (fs *FilerServer) updateFilerStore(ctx context.Context, r *http.Request, w http.ResponseWriter, replication string,
+	collection string, ret *operation.UploadResult, md5value []byte, fileId string, ttlSeconds int32) (err error) {
 
 	stats.FilerRequestCounter.WithLabelValues("postStoreWrite").Inc()
 	start := time.Now()
@@ -186,6 +186,7 @@ func (fs *FilerServer) updateFilerStore(ctx context.Context, r *http.Request, w
 			Collection:  collection,
 			TtlSec:      ttlSeconds,
 			Mime:        ret.Mime,
+			Md5:         md5value,
 		},
 		Chunks: []*filer_pb.FileChunk{{
 			FileId: fileId,
@@ -212,15 +213,15 @@ func (fs *FilerServer) updateFilerStore(ctx context.Context, r *http.Request, w
 }
 
 // send request to volume server
-func (fs *FilerServer) uploadToVolumeServer(r *http.Request, u *url.URL, auth security.EncodedJwt, w http.ResponseWriter, fileId string) (ret *operation.UploadResult, err error) {
+func (fs *FilerServer) uploadToVolumeServer(r *http.Request, u *url.URL, auth security.EncodedJwt, w http.ResponseWriter, fileId string) (ret *operation.UploadResult, md5value []byte, err error) {
 
 	stats.FilerRequestCounter.WithLabelValues("postUpload").Inc()
 	start := time.Now()
 	defer func() { stats.FilerRequestHistogram.WithLabelValues("postUpload").Observe(time.Since(start).Seconds()) }()
 
 	ret = &operation.UploadResult{}
-	hash := md5.New()
-	var body = ioutil.NopCloser(io.TeeReader(r.Body, hash))
+	md5Hash := md5.New()
+	var body = ioutil.NopCloser(io.TeeReader(r.Body, md5Hash))
 
 	request := &http.Request{
 		Method:        r.Method,
@@ -285,7 +286,8 @@ func (fs *FilerServer) uploadToVolumeServer(r *http.Request, u *url.URL, auth se
 		}
 	}
 	// use filer calculated md5 ETag, instead of the volume server crc ETag
-	ret.ETag = fmt.Sprintf("%x", hash.Sum(nil))
+	md5value = md5Hash.Sum(nil)
+	ret.ETag = fmt.Sprintf("%x", md5value)
 	return
 }
 
diff --git a/weed/server/filer_server_handlers_write_autochunk.go b/weed/server/filer_server_handlers_write_autochunk.go
index e70826b5d..d86530bb8 100644
--- a/weed/server/filer_server_handlers_write_autochunk.go
+++ b/weed/server/filer_server_handlers_write_autochunk.go
@@ -2,7 +2,9 @@ package weed_server
 
 import (
 	"context"
+	"crypto/md5"
 	"io"
+	"io/ioutil"
 	"net/http"
 	"path"
 	"strconv"
@@ -91,10 +93,13 @@ func (fs *FilerServer) doAutoChunk(ctx context.Context, w http.ResponseWriter, r
 
 	var fileChunks []*filer_pb.FileChunk
 
+	md5Hash := md5.New()
+	var partReader = ioutil.NopCloser(io.TeeReader(part1, md5Hash))
+
 	chunkOffset := int64(0)
 
 	for chunkOffset < contentLength {
-		limitedReader := io.LimitReader(part1, int64(chunkSize))
+		limitedReader := io.LimitReader(partReader, int64(chunkSize))
 
 		// assign one file id for one chunk
 		fileId, urlLocation, auth, assignErr := fs.assignNewFileInfo(w, r, replication, collection, dataCenter, ttlString)
@@ -157,6 +162,7 @@ func (fs *FilerServer) doAutoChunk(ctx context.Context, w http.ResponseWriter, r
 			Collection:  collection,
 			TtlSec:      ttlSec,
 			Mime:        contentType,
+			Md5:         md5Hash.Sum(nil),
 		},
 		Chunks: fileChunks,
 	}

From dc08e4098ffbb3d7d95c9decac7407e67349baa9 Mon Sep 17 00:00:00 2001
From: Chris Lu <chris.lu@gmail.com>
Date: Wed, 8 Apr 2020 09:13:26 -0700
Subject: [PATCH 07/81] add etag only for PUT or large chunked uploads

---
 weed/filer2/filechunks.go                   |  4 ++--
 weed/server/filer_server_handlers_write.go  | 13 ++++++++++---
 weed/server/volume_server_handlers_write.go |  8 ++++++++
 3 files changed, 20 insertions(+), 5 deletions(-)

diff --git a/weed/filer2/filechunks.go b/weed/filer2/filechunks.go
index e8e4c305b..48eaeea27 100644
--- a/weed/filer2/filechunks.go
+++ b/weed/filer2/filechunks.go
@@ -22,14 +22,14 @@ func TotalSize(chunks []*filer_pb.FileChunk) (size uint64) {
 
 func ETag(entry *filer_pb.Entry) (etag string) {
 	if entry.Attributes == nil || entry.Attributes.Md5 == nil {
-		ETagChunks(entry.Chunks)
+		return ETagChunks(entry.Chunks)
 	}
 	return fmt.Sprintf("%x", entry.Attributes.Md5)
 }
 
 func ETagEntry(entry *Entry) (etag string) {
 	if entry.Attr.Md5 == nil {
-		ETagChunks(entry.Chunks)
+		return ETagChunks(entry.Chunks)
 	}
 	return fmt.Sprintf("%x", entry.Attr.Md5)
 }
diff --git a/weed/server/filer_server_handlers_write.go b/weed/server/filer_server_handlers_write.go
index 33e6ee043..ce184875c 100644
--- a/weed/server/filer_server_handlers_write.go
+++ b/weed/server/filer_server_handlers_write.go
@@ -220,8 +220,13 @@ func (fs *FilerServer) uploadToVolumeServer(r *http.Request, u *url.URL, auth se
 	defer func() { stats.FilerRequestHistogram.WithLabelValues("postUpload").Observe(time.Since(start).Seconds()) }()
 
 	ret = &operation.UploadResult{}
+
 	md5Hash := md5.New()
-	var body = ioutil.NopCloser(io.TeeReader(r.Body, md5Hash))
+	body := r.Body
+	if r.Method == "PUT" {
+		// only PUT or large chunked files has Md5 in attributes
+		body = ioutil.NopCloser(io.TeeReader(r.Body, md5Hash))
+	}
 
 	request := &http.Request{
 		Method:        r.Method,
@@ -286,8 +291,10 @@ func (fs *FilerServer) uploadToVolumeServer(r *http.Request, u *url.URL, auth se
 		}
 	}
 	// use filer calculated md5 ETag, instead of the volume server crc ETag
-	md5value = md5Hash.Sum(nil)
-	ret.ETag = fmt.Sprintf("%x", md5value)
+	if r.Method == "PUT" {
+		md5value = md5Hash.Sum(nil)
+	}
+	ret.ETag = getEtag(resp)
 	return
 }
 
diff --git a/weed/server/volume_server_handlers_write.go b/weed/server/volume_server_handlers_write.go
index 3a71a1332..968ece11f 100644
--- a/weed/server/volume_server_handlers_write.go
+++ b/weed/server/volume_server_handlers_write.go
@@ -166,3 +166,11 @@ func setEtag(w http.ResponseWriter, etag string) {
 		}
 	}
 }
+
+func getEtag(resp *http.Response) (etag string){
+	etag = resp.Header.Get("ETag")
+	if strings.HasPrefix(etag, "\"") && strings.HasSuffix(etag, "\""){
+		return etag[1:len(etag)-1]
+	}
+	return
+}

From bd56172b829d447d320cd6933b6d72363dae7dac Mon Sep 17 00:00:00 2001
From: Chris Lu <chris.lu@gmail.com>
Date: Wed, 8 Apr 2020 12:50:20 -0700
Subject: [PATCH 08/81] simplify file handle management

---
 weed/filesys/wfs.go | 36 +++++++++++-------------------------
 1 file changed, 11 insertions(+), 25 deletions(-)

diff --git a/weed/filesys/wfs.go b/weed/filesys/wfs.go
index 590c39790..191b79845 100644
--- a/weed/filesys/wfs.go
+++ b/weed/filesys/wfs.go
@@ -54,9 +54,8 @@ type WFS struct {
 	listDirectoryEntriesCache *ccache.Cache
 
 	// contains all open handles, protected by handlesLock
-	handlesLock       sync.Mutex
-	handles           []*FileHandle
-	pathToHandleIndex map[util.FullPath]int
+	handlesLock sync.Mutex
+	handles     map[uint64]*FileHandle
 
 	bufPool sync.Pool
 
@@ -76,7 +75,7 @@ func NewSeaweedFileSystem(option *Option) *WFS {
 	wfs := &WFS{
 		option:                    option,
 		listDirectoryEntriesCache: ccache.New(ccache.Configure().MaxSize(option.DirListCacheLimit * 3).ItemsToPrune(100)),
-		pathToHandleIndex:         make(map[util.FullPath]int),
+		handles:                   make(map[uint64]*FileHandle),
 		bufPool: sync.Pool{
 			New: func() interface{} {
 				return make([]byte, option.ChunkSizeLimit)
@@ -117,26 +116,15 @@ func (wfs *WFS) AcquireHandle(file *File, uid, gid uint32) (fileHandle *FileHand
 	wfs.handlesLock.Lock()
 	defer wfs.handlesLock.Unlock()
 
-	index, found := wfs.pathToHandleIndex[fullpath]
-	if found && wfs.handles[index] != nil {
-		glog.V(2).Infoln(fullpath, "found fileHandle id", index)
-		return wfs.handles[index]
+	inodeId := file.fullpath().AsInode()
+	existingHandle, found := wfs.handles[inodeId]
+	if found && existingHandle != nil {
+		return existingHandle
 	}
 
 	fileHandle = newFileHandle(file, uid, gid)
-	for i, h := range wfs.handles {
-		if h == nil {
-			wfs.handles[i] = fileHandle
-			fileHandle.handle = uint64(i)
-			wfs.pathToHandleIndex[fullpath] = i
-			glog.V(4).Infof("%s reuse fh %d", fullpath, fileHandle.handle)
-			return
-		}
-	}
-
-	wfs.handles = append(wfs.handles, fileHandle)
-	fileHandle.handle = uint64(len(wfs.handles) - 1)
-	wfs.pathToHandleIndex[fullpath] = int(fileHandle.handle)
+	wfs.handles[inodeId] = fileHandle
+	fileHandle.handle = inodeId
 	glog.V(4).Infof("%s new fh %d", fullpath, fileHandle.handle)
 
 	return
@@ -147,10 +135,8 @@ func (wfs *WFS) ReleaseHandle(fullpath util.FullPath, handleId fuse.HandleID) {
 	defer wfs.handlesLock.Unlock()
 
 	glog.V(4).Infof("%s ReleaseHandle id %d current handles length %d", fullpath, handleId, len(wfs.handles))
-	delete(wfs.pathToHandleIndex, fullpath)
-	if int(handleId) < len(wfs.handles) {
-		wfs.handles[int(handleId)] = nil
-	}
+
+	delete(wfs.handles, fullpath.AsInode())
 
 	return
 }

From b524a40375d322900e65ba60cc924efe761c660b Mon Sep 17 00:00:00 2001
From: Chris Lu <chris.lu@gmail.com>
Date: Wed, 8 Apr 2020 12:50:34 -0700
Subject: [PATCH 09/81] add locking to fs cache

---
 weed/filesys/fscache.go | 33 +++++++++++++++++++++++++++++++--
 1 file changed, 31 insertions(+), 2 deletions(-)

diff --git a/weed/filesys/fscache.go b/weed/filesys/fscache.go
index ca8c7de5b..831f3808c 100644
--- a/weed/filesys/fscache.go
+++ b/weed/filesys/fscache.go
@@ -9,6 +9,7 @@ import (
 
 type FsCache struct {
 	root *FsNode
+	sync.RWMutex
 }
 type FsNode struct {
 	parent       *FsNode
@@ -27,6 +28,14 @@ func newFsCache(root fs.Node) *FsCache {
 }
 
 func (c *FsCache) GetFsNode(path util.FullPath) fs.Node {
+
+	c.RLock()
+	defer c.RUnlock()
+
+	return c.doGetFsNode(path)
+}
+
+func (c *FsCache) doGetFsNode(path util.FullPath) fs.Node {
 	t := c.root
 	for _, p := range path.Split() {
 		t = t.findChild(p)
@@ -38,6 +47,14 @@ func (c *FsCache) GetFsNode(path util.FullPath) fs.Node {
 }
 
 func (c *FsCache) SetFsNode(path util.FullPath, node fs.Node) {
+
+	c.Lock()
+	defer c.Unlock()
+
+	c.doSetFsNode(path, node)
+}
+
+func (c *FsCache) doSetFsNode(path util.FullPath, node fs.Node) {
 	t := c.root
 	for _, p := range path.Split() {
 		t = t.ensureChild(p)
@@ -45,17 +62,26 @@ func (c *FsCache) SetFsNode(path util.FullPath, node fs.Node) {
 	t.node = node
 }
 
+
 func (c *FsCache) EnsureFsNode(path util.FullPath, genNodeFn func() fs.Node) fs.Node {
-	t := c.GetFsNode(path)
+
+	c.Lock()
+	defer c.Unlock()
+
+	t := c.doGetFsNode(path)
 	if t != nil {
 		return t
 	}
 	t = genNodeFn()
-	c.SetFsNode(path, t)
+	c.doSetFsNode(path, t)
 	return t
 }
 
 func (c *FsCache) DeleteFsNode(path util.FullPath) {
+
+	c.Lock()
+	defer c.Unlock()
+
 	t := c.root
 	for _, p := range path.Split() {
 		t = t.findChild(p)
@@ -72,6 +98,9 @@ func (c *FsCache) DeleteFsNode(path util.FullPath) {
 // oldPath and newPath are full path including the new name
 func (c *FsCache) Move(oldPath util.FullPath, newPath util.FullPath) *FsNode {
 
+	c.Lock()
+	defer c.Unlock()
+
 	// find old node
 	src := c.root
 	for _, p := range oldPath.Split() {

From 6630541399afd64ecce76b0197f0562ccf7f4da7 Mon Sep 17 00:00:00 2001
From: Chris Lu <chris.lu@gmail.com>
Date: Wed, 8 Apr 2020 12:50:59 -0700
Subject: [PATCH 10/81] ensure correct file size when opening existing files

---
 weed/filesys/filehandle.go | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/weed/filesys/filehandle.go b/weed/filesys/filehandle.go
index 83a93c062..18544a7f1 100644
--- a/weed/filesys/filehandle.go
+++ b/weed/filesys/filehandle.go
@@ -33,12 +33,16 @@ type FileHandle struct {
 }
 
 func newFileHandle(file *File, uid, gid uint32) *FileHandle {
-	return &FileHandle{
+	fh := &FileHandle{
 		f:          file,
 		dirtyPages: newDirtyPages(file),
 		Uid:        uid,
 		Gid:        gid,
 	}
+	if fh.f.entry != nil {
+		fh.f.entry.Attributes.FileSize = filer2.TotalSize(fh.f.entry.Chunks)
+	}
+	return fh
 }
 
 var _ = fs.Handle(&FileHandle{})

From ed98223b08dcb192aa047864af4aa67874e0a54e Mon Sep 17 00:00:00 2001
From: Chris Lu <chris.lu@gmail.com>
Date: Wed, 8 Apr 2020 20:32:57 -0700
Subject: [PATCH 11/81] still log, but not persisting the changes

---
 weed/filer2/filer_notify.go | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/weed/filer2/filer_notify.go b/weed/filer2/filer_notify.go
index c57631466..095587038 100644
--- a/weed/filer2/filer_notify.go
+++ b/weed/filer2/filer_notify.go
@@ -45,9 +45,7 @@ func (f *Filer) NotifyUpdateEvent(oldEntry, newEntry *Entry, deleteChunks bool)
 		notification.Queue.SendMessage(fullpath, eventNotification)
 	}
 
-	if false {
-		f.logMetaEvent(time.Now(), fullpath, eventNotification)
-	}
+	f.logMetaEvent(time.Now(), fullpath, eventNotification)
 
 }
 
@@ -70,6 +68,9 @@ func (f *Filer) logMetaEvent(ts time.Time, fullpath string, eventNotification *f
 }
 
 func (f *Filer) logFlushFunc(startTime, stopTime time.Time, buf []byte) {
+
+	return
+
 	targetFile := fmt.Sprintf("/.meta/log/%04d/%02d/%02d/%02d/%02d/%02d.%09d.log",
 		startTime.Year(), startTime.Month(), startTime.Day(), startTime.Hour(), startTime.Minute(),
 		startTime.Second(), startTime.Nanosecond())

From 00b993a234038e0c70eab4f307ae9c89e6771a83 Mon Sep 17 00:00:00 2001
From: Chris Lu <chris.lu@gmail.com>
Date: Wed, 8 Apr 2020 22:30:38 -0700
Subject: [PATCH 12/81] add util for md5

---
 weed/util/bytes.go | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/weed/util/bytes.go b/weed/util/bytes.go
index d9e462693..d72d199f8 100644
--- a/weed/util/bytes.go
+++ b/weed/util/bytes.go
@@ -2,6 +2,7 @@ package util
 
 import (
 	"crypto/md5"
+	"fmt"
 	"io"
 )
 
@@ -91,3 +92,9 @@ func HashToInt32(data []byte) (v int32) {
 
 	return
 }
+
+func Md5(data []byte) string {
+	hash := md5.New()
+	hash.Write(data)
+	return fmt.Sprintf("%x", hash.Sum(nil))
+}

From 006f78b70b458ca55c66f9f50692cbd9996fb5ad Mon Sep 17 00:00:00 2001
From: Chris Lu <chris.lu@gmail.com>
Date: Wed, 8 Apr 2020 22:31:19 -0700
Subject: [PATCH 13/81] make a copy of request data

to avoid concurrency issues
---
 weed/filesys/filehandle.go | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/weed/filesys/filehandle.go b/weed/filesys/filehandle.go
index 18544a7f1..ea53ed4c0 100644
--- a/weed/filesys/filehandle.go
+++ b/weed/filesys/filehandle.go
@@ -114,21 +114,23 @@ func (fh *FileHandle) readFromChunks(buff []byte, offset int64) (int64, error) {
 func (fh *FileHandle) Write(ctx context.Context, req *fuse.WriteRequest, resp *fuse.WriteResponse) error {
 
 	// write the request to volume servers
+	data := make([]byte, len(req.Data))
+	copy(data, req.Data)
 
-	fh.f.entry.Attributes.FileSize = uint64(max(req.Offset+int64(len(req.Data)), int64(fh.f.entry.Attributes.FileSize)))
+	fh.f.entry.Attributes.FileSize = uint64(max(req.Offset+int64(len(data)), int64(fh.f.entry.Attributes.FileSize)))
 	// glog.V(0).Infof("%v write [%d,%d)", fh.f.fullpath(), req.Offset, req.Offset+int64(len(req.Data)))
 
-	chunks, err := fh.dirtyPages.AddPage(req.Offset, req.Data)
+	chunks, err := fh.dirtyPages.AddPage(req.Offset, data)
 	if err != nil {
-		glog.Errorf("%v write fh %d: [%d,%d): %v", fh.f.fullpath(), fh.handle, req.Offset, req.Offset+int64(len(req.Data)), err)
+		glog.Errorf("%v write fh %d: [%d,%d): %v", fh.f.fullpath(), fh.handle, req.Offset, req.Offset+int64(len(data)), err)
 		return fuse.EIO
 	}
 
-	resp.Size = len(req.Data)
+	resp.Size = len(data)
 
 	if req.Offset == 0 {
 		// detect mime type
-		detectedMIME := mimetype.Detect(req.Data)
+		detectedMIME := mimetype.Detect(data)
 		fh.contentType = detectedMIME.String()
 		if ext := path.Ext(fh.f.Name); ext != detectedMIME.Extension() {
 			fh.contentType = mime.TypeByExtension(ext)
@@ -159,6 +161,8 @@ func (fh *FileHandle) Release(ctx context.Context, req *fuse.ReleaseRequest) err
 	}
 	fh.f.entryViewCache = nil
 	fh.f.reader = nil
+	fh.dirtyPages = nil
+	fh.f = nil
 
 	return nil
 }

From 8a7327660280dfc4efe4b18b68a47eee235189fe Mon Sep 17 00:00:00 2001
From: Chris Lu <chris.lu@gmail.com>
Date: Wed, 8 Apr 2020 22:33:36 -0700
Subject: [PATCH 14/81] refactor a bit

---
 weed/operation/upload_content.go | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/weed/operation/upload_content.go b/weed/operation/upload_content.go
index 14e0f7cd4..f6f3d0dcc 100644
--- a/weed/operation/upload_content.go
+++ b/weed/operation/upload_content.go
@@ -45,11 +45,9 @@ var fileNameEscaper = strings.NewReplacer("\\", "\\\\", "\"", "\\\"")
 
 // Upload sends a POST request to a volume server to upload the content with adjustable compression level
 func UploadData(uploadUrl string, filename string, cipher bool, data []byte, isInputGzipped bool, mtype string, pairMap map[string]string, jwt security.EncodedJwt) (uploadResult *UploadResult, err error) {
-	hash := md5.New()
-	hash.Write(data)
 	uploadResult, err = doUploadData(uploadUrl, filename, cipher, data, isInputGzipped, mtype, pairMap, jwt)
 	if uploadResult != nil {
-		uploadResult.Md5 = fmt.Sprintf("%x", hash.Sum(nil))
+		uploadResult.Md5 = util.Md5(data)
 	}
 	return
 }

From 8764bdb9df65b2061b56640c6583ab076ae7d70e Mon Sep 17 00:00:00 2001
From: Chris Lu <chris.lu@gmail.com>
Date: Wed, 8 Apr 2020 22:35:14 -0700
Subject: [PATCH 15/81] 1.72

---
 k8s/seaweedfs/Chart.yaml  | 2 +-
 k8s/seaweedfs/values.yaml | 2 +-
 weed/util/constants.go    | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/k8s/seaweedfs/Chart.yaml b/k8s/seaweedfs/Chart.yaml
index d606c6bf8..2986fc4cf 100644
--- a/k8s/seaweedfs/Chart.yaml
+++ b/k8s/seaweedfs/Chart.yaml
@@ -1,4 +1,4 @@
 apiVersion: v1
 description: SeaweedFS
 name: seaweedfs
-version: 1.71
+version: 1.72
diff --git a/k8s/seaweedfs/values.yaml b/k8s/seaweedfs/values.yaml
index e1b102088..b572de823 100644
--- a/k8s/seaweedfs/values.yaml
+++ b/k8s/seaweedfs/values.yaml
@@ -4,7 +4,7 @@ global:
   registry: ""
   repository: ""
   imageName: chrislusf/seaweedfs
-  imageTag: "1.71"
+  imageTag: "1.72"
   imagePullPolicy: IfNotPresent
   imagePullSecrets: imagepullsecret
   restartPolicy: Always
diff --git a/weed/util/constants.go b/weed/util/constants.go
index 6d1498164..81b6f6a60 100644
--- a/weed/util/constants.go
+++ b/weed/util/constants.go
@@ -5,5 +5,5 @@ import (
 )
 
 var (
-	VERSION = fmt.Sprintf("%s %d.%d", sizeLimit, 1, 71)
+	VERSION = fmt.Sprintf("%s %d.%d", sizeLimit, 1, 72)
 )

From bb78ab991553fed64116b673836452c831044702 Mon Sep 17 00:00:00 2001
From: Chris Lu <chris.lu@gmail.com>
Date: Wed, 8 Apr 2020 23:12:37 -0700
Subject: [PATCH 16/81] wait for master to be aware of the failed volumes

fix https://github.com/chrislusf/seaweedfs/issues/1268
---
 weed/filesys/dirty_page.go | 18 ++++++++++--------
 1 file changed, 10 insertions(+), 8 deletions(-)

diff --git a/weed/filesys/dirty_page.go b/weed/filesys/dirty_page.go
index e2e628407..77eae927e 100644
--- a/weed/filesys/dirty_page.go
+++ b/weed/filesys/dirty_page.go
@@ -125,16 +125,18 @@ func (pages *ContinuousDirtyPages) saveExistingLargestPageToStorage() (chunk *fi
 		return nil, false, nil
 	}
 
-	chunk, err = pages.saveToStorage(maxList.ToReader(), maxList.Offset(), maxList.Size())
-	if err == nil {
-		hasSavedData = true
-		glog.V(3).Infof("%s saveToStorage [%d,%d) %s", pages.f.fullpath(), maxList.Offset(), maxList.Offset()+maxList.Size(), chunk.FileId)
-	} else {
-		glog.V(0).Infof("%s saveToStorage [%d,%d): %v", pages.f.fullpath(), maxList.Offset(), maxList.Offset()+maxList.Size(), err)
-		return
+	for {
+		chunk, err = pages.saveToStorage(maxList.ToReader(), maxList.Offset(), maxList.Size())
+		if err == nil {
+			hasSavedData = true
+			glog.V(3).Infof("%s saveToStorage [%d,%d) %s", pages.f.fullpath(), maxList.Offset(), maxList.Offset()+maxList.Size(), chunk.FileId)
+			time.Sleep(5 * time.Second)
+		} else {
+			glog.V(0).Infof("%s saveToStorage [%d,%d): %v", pages.f.fullpath(), maxList.Offset(), maxList.Offset()+maxList.Size(), err)
+			return
+		}
 	}
 
-	return
 }
 
 func (pages *ContinuousDirtyPages) saveToStorage(reader io.Reader, offset int64, size int64) (*filer_pb.FileChunk, error) {

From 2329d9e0c18aa2dbc2b843a43fe269ee539a0f39 Mon Sep 17 00:00:00 2001
From: Chris Lu <chris.lu@gmail.com>
Date: Wed, 8 Apr 2020 23:36:22 -0700
Subject: [PATCH 17/81] add volume.fix.replication to default scaffold

---
 weed/command/scaffold.go | 1 +
 1 file changed, 1 insertion(+)

diff --git a/weed/command/scaffold.go b/weed/command/scaffold.go
index 72a36bb54..f2832510c 100644
--- a/weed/command/scaffold.go
+++ b/weed/command/scaffold.go
@@ -359,6 +359,7 @@ scripts = """
   ec.rebuild -force
   ec.balance -force
   volume.balance -force
+  volume.fix.replication
 """
 sleep_minutes = 17          # sleep minutes between each script execution
 

From f6a7e79dc370dfb2e3fe5bff6a380afde95b9a7f Mon Sep 17 00:00:00 2001
From: Chris Lu <chris.lu@gmail.com>
Date: Wed, 8 Apr 2020 23:57:15 -0700
Subject: [PATCH 18/81] weed shell: simplify CLI option for filer

---
 weed/command/scaffold.go     |  3 ++-
 weed/command/shell.go        | 15 ++++++++-------
 weed/server/master_server.go |  9 +++++----
 weed/util/parse.go           | 16 ++++++++++++++++
 4 files changed, 31 insertions(+), 12 deletions(-)

diff --git a/weed/command/scaffold.go b/weed/command/scaffold.go
index f2832510c..2aad29548 100644
--- a/weed/command/scaffold.go
+++ b/weed/command/scaffold.go
@@ -364,7 +364,8 @@ scripts = """
 sleep_minutes = 17          # sleep minutes between each script execution
 
 [master.filer]
-default_filer_url = "http://localhost:8888/"
+default = "localhost:8888"    # used by maintenance scripts if the scripts needs to use fs related commands
+
 
 [master.sequencer]
 type = "memory"     # Choose [memory|etcd] type for storing the file id sequence
diff --git a/weed/command/shell.go b/weed/command/shell.go
index dcf70608f..6dd768f47 100644
--- a/weed/command/shell.go
+++ b/weed/command/shell.go
@@ -9,14 +9,14 @@ import (
 )
 
 var (
-	shellOptions         shell.ShellOptions
-	shellInitialFilerUrl *string
+	shellOptions      shell.ShellOptions
+	shellInitialFiler *string
 )
 
 func init() {
 	cmdShell.Run = runShell // break init cycle
 	shellOptions.Masters = cmdShell.Flag.String("master", "localhost:9333", "comma-separated master servers")
-	shellInitialFilerUrl = cmdShell.Flag.String("filer.url", "http://localhost:8888/", "initial filer url")
+	shellInitialFiler = cmdShell.Flag.String("filer", "localhost:8888", "filer host and port")
 }
 
 var cmdShell = &Command{
@@ -32,12 +32,13 @@ func runShell(command *Command, args []string) bool {
 	util.LoadConfiguration("security", false)
 	shellOptions.GrpcDialOption = security.LoadClientTLS(util.GetViper(), "grpc.client")
 
-	var filerPwdErr error
-	shellOptions.FilerHost, shellOptions.FilerPort, shellOptions.Directory, filerPwdErr = util.ParseFilerUrl(*shellInitialFilerUrl)
-	if filerPwdErr != nil {
-		fmt.Printf("failed to parse url filer.url=%s : %v\n", *shellInitialFilerUrl, filerPwdErr)
+	var err error
+	shellOptions.FilerHost, shellOptions.FilerPort, err = util.ParseHostPort(*shellInitialFiler)
+	if err != nil {
+		fmt.Printf("failed to parse filer %s: %v\n", *shellInitialFiler, err)
 		return false
 	}
+	shellOptions.Directory = "/"
 
 	shell.RunShell(shellOptions)
 
diff --git a/weed/server/master_server.go b/weed/server/master_server.go
index 497990f29..4a264d432 100644
--- a/weed/server/master_server.go
+++ b/weed/server/master_server.go
@@ -197,8 +197,8 @@ func (ms *MasterServer) startAdminScripts() {
 	v.SetDefault("master.maintenance.sleep_minutes", 17)
 	sleepMinutes := v.GetInt("master.maintenance.sleep_minutes")
 
-	v.SetDefault("master.filer.default_filer_url", "http://localhost:8888/")
-	filerURL := v.GetString("master.filer.default_filer_url")
+	v.SetDefault("master.filer.default", "localhost:8888")
+	filerHostPort := v.GetString("master.filer.default")
 
 	scriptLines := strings.Split(adminScripts, "\n")
 
@@ -208,9 +208,10 @@ func (ms *MasterServer) startAdminScripts() {
 	shellOptions.GrpcDialOption = security.LoadClientTLS(v, "grpc.master")
 	shellOptions.Masters = &masterAddress
 
-	shellOptions.FilerHost, shellOptions.FilerPort, shellOptions.Directory, err = util.ParseFilerUrl(filerURL)
+	shellOptions.FilerHost, shellOptions.FilerPort, err = util.ParseHostPort(filerHostPort)
+	shellOptions.Directory = "/"
 	if err != nil {
-		glog.V(0).Infof("failed to parse master.filer.default_filer_urll=%s : %v\n", filerURL, err)
+		glog.V(0).Infof("failed to parse master.filer.default = %s : %v\n", filerHostPort, err)
 		return
 	}
 
diff --git a/weed/util/parse.go b/weed/util/parse.go
index 6593d43b6..0955db682 100644
--- a/weed/util/parse.go
+++ b/weed/util/parse.go
@@ -1,6 +1,7 @@
 package util
 
 import (
+	"fmt"
 	"net/url"
 	"strconv"
 	"strings"
@@ -45,3 +46,18 @@ func ParseFilerUrl(entryPath string) (filerServer string, filerPort int64, path
 	path = u.Path
 	return
 }
+
+func ParseHostPort(hostPort string) (filerServer string, filerPort int64, err error) {
+	parts := strings.Split(hostPort, ":")
+	if len(parts) != 2 {
+		err = fmt.Errorf("failed to parse %s\n", hostPort)
+		return
+	}
+
+	filerPort, err = strconv.ParseInt(parts[1], 10, 64)
+	if err == nil {
+		filerServer = parts[0]
+	}
+
+	return
+}

From 59f40e202783dae6c83421e278608eeda7a97dbf Mon Sep 17 00:00:00 2001
From: Chris Lu <chris.lu@gmail.com>
Date: Thu, 9 Apr 2020 00:26:24 -0700
Subject: [PATCH 19/81] volume: best effort to detect ip address

fix https://github.com/chrislusf/seaweedfs/issues/1264
---
 weed/command/volume.go |  2 +-
 weed/util/network.go   | 25 +++++++++++++++++++++++++
 2 files changed, 26 insertions(+), 1 deletion(-)
 create mode 100644 weed/util/network.go

diff --git a/weed/command/volume.go b/weed/command/volume.go
index 68a0ce223..341111ecd 100644
--- a/weed/command/volume.go
+++ b/weed/command/volume.go
@@ -127,7 +127,7 @@ func (v VolumeServerOptions) startVolumeServer(volumeFolders, maxVolumeCounts, v
 	}
 
 	if *v.ip == "" {
-		*v.ip = "127.0.0.1"
+		*v.ip = util.DetectedHostAddress()
 	}
 
 	if *v.publicPort == 0 {
diff --git a/weed/util/network.go b/weed/util/network.go
new file mode 100644
index 000000000..7108cfea6
--- /dev/null
+++ b/weed/util/network.go
@@ -0,0 +1,25 @@
+package util
+
+import (
+	"net"
+
+	"github.com/chrislusf/seaweedfs/weed/glog"
+)
+
+func DetectedHostAddress() string {
+	addrs, err := net.InterfaceAddrs()
+	if err != nil {
+		glog.V(0).Infof("failed to detect ip address: %v", err)
+		return ""
+	}
+
+	for _, a := range addrs {
+		if ipnet, ok := a.(*net.IPNet); ok && !ipnet.IP.IsLoopback() {
+			if ipnet.IP.To4() != nil {
+				return ipnet.IP.String()
+			}
+		}
+	}
+
+	return "localhost"
+}

From 56ec8c45131e425ca7e474c7f36ddb0c768b3e02 Mon Sep 17 00:00:00 2001
From: Chris Lu <chris.lu@gmail.com>
Date: Thu, 9 Apr 2020 21:43:05 -0700
Subject: [PATCH 20/81] fix to avoid nil file an dirty pages

---
 weed/filesys/filehandle.go | 2 --
 1 file changed, 2 deletions(-)

diff --git a/weed/filesys/filehandle.go b/weed/filesys/filehandle.go
index ea53ed4c0..8b1be5209 100644
--- a/weed/filesys/filehandle.go
+++ b/weed/filesys/filehandle.go
@@ -161,8 +161,6 @@ func (fh *FileHandle) Release(ctx context.Context, req *fuse.ReleaseRequest) err
 	}
 	fh.f.entryViewCache = nil
 	fh.f.reader = nil
-	fh.dirtyPages = nil
-	fh.f = nil
 
 	return nil
 }

From eb39df27048820f405ab4b5ecdc96047b9274130 Mon Sep 17 00:00:00 2001
From: Chris Lu <chris.lu@gmail.com>
Date: Thu, 9 Apr 2020 21:43:26 -0700
Subject: [PATCH 21/81] avoid dead loop

---
 weed/filesys/dirty_page.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/weed/filesys/dirty_page.go b/weed/filesys/dirty_page.go
index 77eae927e..ce74d64d5 100644
--- a/weed/filesys/dirty_page.go
+++ b/weed/filesys/dirty_page.go
@@ -130,10 +130,10 @@ func (pages *ContinuousDirtyPages) saveExistingLargestPageToStorage() (chunk *fi
 		if err == nil {
 			hasSavedData = true
 			glog.V(3).Infof("%s saveToStorage [%d,%d) %s", pages.f.fullpath(), maxList.Offset(), maxList.Offset()+maxList.Size(), chunk.FileId)
-			time.Sleep(5 * time.Second)
+			return
 		} else {
 			glog.V(0).Infof("%s saveToStorage [%d,%d): %v", pages.f.fullpath(), maxList.Offset(), maxList.Offset()+maxList.Size(), err)
-			return
+			time.Sleep(5 * time.Second)
 		}
 	}
 

From 43c62a8a954cc34f3120d520ab87459db307188c Mon Sep 17 00:00:00 2001
From: Chris Lu <chris.lu@gmail.com>
Date: Thu, 9 Apr 2020 21:44:30 -0700
Subject: [PATCH 22/81] 1.73

---
 k8s/seaweedfs/Chart.yaml  | 2 +-
 k8s/seaweedfs/values.yaml | 2 +-
 weed/util/constants.go    | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/k8s/seaweedfs/Chart.yaml b/k8s/seaweedfs/Chart.yaml
index 2986fc4cf..30a3ca83e 100644
--- a/k8s/seaweedfs/Chart.yaml
+++ b/k8s/seaweedfs/Chart.yaml
@@ -1,4 +1,4 @@
 apiVersion: v1
 description: SeaweedFS
 name: seaweedfs
-version: 1.72
+version: 1.73
diff --git a/k8s/seaweedfs/values.yaml b/k8s/seaweedfs/values.yaml
index b572de823..c46fd2a3f 100644
--- a/k8s/seaweedfs/values.yaml
+++ b/k8s/seaweedfs/values.yaml
@@ -4,7 +4,7 @@ global:
   registry: ""
   repository: ""
   imageName: chrislusf/seaweedfs
-  imageTag: "1.72"
+  imageTag: "1.73"
   imagePullPolicy: IfNotPresent
   imagePullSecrets: imagepullsecret
   restartPolicy: Always
diff --git a/weed/util/constants.go b/weed/util/constants.go
index 81b6f6a60..0fd43e318 100644
--- a/weed/util/constants.go
+++ b/weed/util/constants.go
@@ -5,5 +5,5 @@ import (
 )
 
 var (
-	VERSION = fmt.Sprintf("%s %d.%d", sizeLimit, 1, 72)
+	VERSION = fmt.Sprintf("%s %d.%d", sizeLimit, 1, 73)
 )

From 8db2120bee6073a5e781271563ddc9996c509719 Mon Sep 17 00:00:00 2001
From: Chris Lu <chris.lu@gmail.com>
Date: Thu, 9 Apr 2020 23:42:59 -0700
Subject: [PATCH 23/81] add logs

---
 weed/command/volume.go | 1 +
 1 file changed, 1 insertion(+)

diff --git a/weed/command/volume.go b/weed/command/volume.go
index 341111ecd..936998d82 100644
--- a/weed/command/volume.go
+++ b/weed/command/volume.go
@@ -128,6 +128,7 @@ func (v VolumeServerOptions) startVolumeServer(volumeFolders, maxVolumeCounts, v
 
 	if *v.ip == "" {
 		*v.ip = util.DetectedHostAddress()
+		glog.V(0).Infof("detected volume server ip address: %v", *v.ip)
 	}
 
 	if *v.publicPort == 0 {

From 9fa065f600cbb4fbdabcca60e12fb144a4ed97b5 Mon Sep 17 00:00:00 2001
From: Chris Lu <chris.lu@gmail.com>
Date: Thu, 9 Apr 2020 23:43:09 -0700
Subject: [PATCH 24/81] typo in logs

---
 weed/storage/store.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/weed/storage/store.go b/weed/storage/store.go
index 84374b174..d6b623e63 100644
--- a/weed/storage/store.go
+++ b/weed/storage/store.go
@@ -421,7 +421,7 @@ func (s *Store) MaybeAdjustVolumeMax() (hasChanges bool) {
 				maxVolumeCount += int(uint64(unclaimedSpaces)/volumeSizeLimit) - 1
 			}
 			diskLocation.MaxVolumeCount = maxVolumeCount
-			glog.V(0).Infof("disk %s max %d unclaimedSpace:%dMB, unused:%dMB volumeSizeLimit:%d/MB",
+			glog.V(0).Infof("disk %s max %d unclaimedSpace:%dMB, unused:%dMB volumeSizeLimit:%dMB",
 				diskLocation.Directory, maxVolumeCount, unclaimedSpaces/1024/1024, unusedSpace/1024/1024, volumeSizeLimit/1024/1024)
 			hasChanges = true
 		}

From bcf37346ef23d2e31d2baa778ff9f839a3af96c0 Mon Sep 17 00:00:00 2001
From: Chris Lu <chris.lu@gmail.com>
Date: Fri, 10 Apr 2020 01:35:59 -0700
Subject: [PATCH 25/81] add timestamp inside lock

---
 weed/filer2/filer_notify.go |  6 +++---
 weed/queue/log_buffer.go    | 42 +++++++++++++++++++------------------
 2 files changed, 25 insertions(+), 23 deletions(-)

diff --git a/weed/filer2/filer_notify.go b/weed/filer2/filer_notify.go
index 095587038..de07e1cf9 100644
--- a/weed/filer2/filer_notify.go
+++ b/weed/filer2/filer_notify.go
@@ -45,11 +45,11 @@ func (f *Filer) NotifyUpdateEvent(oldEntry, newEntry *Entry, deleteChunks bool)
 		notification.Queue.SendMessage(fullpath, eventNotification)
 	}
 
-	f.logMetaEvent(time.Now(), fullpath, eventNotification)
+	f.logMetaEvent(fullpath, eventNotification)
 
 }
 
-func (f *Filer) logMetaEvent(ts time.Time, fullpath string, eventNotification *filer_pb.EventNotification) {
+func (f *Filer) logMetaEvent(fullpath string, eventNotification *filer_pb.EventNotification) {
 
 	dir, _ := util.FullPath(fullpath).DirAndName()
 
@@ -63,7 +63,7 @@ func (f *Filer) logMetaEvent(ts time.Time, fullpath string, eventNotification *f
 		return
 	}
 
-	f.metaLogBuffer.AddToBuffer(ts, []byte(dir), data)
+	f.metaLogBuffer.AddToBuffer([]byte(dir), data)
 
 }
 
diff --git a/weed/queue/log_buffer.go b/weed/queue/log_buffer.go
index 5f8db140d..f24429479 100644
--- a/weed/queue/log_buffer.go
+++ b/weed/queue/log_buffer.go
@@ -46,8 +46,18 @@ func NewLogBuffer(flushInterval time.Duration, flushFn func(startTime, stopTime
 	return lb
 }
 
-func (m *LogBuffer) AddToBuffer(ts time.Time, key, data []byte) {
+func (m *LogBuffer) AddToBuffer(key, data []byte) {
 
+	m.Lock()
+	defer func() {
+		m.Unlock()
+		if m.notifyFn != nil {
+			m.notifyFn()
+		}
+	}()
+
+	// need to put the timestamp inside the lock
+	ts := time.Now()
 	logEntry := &filer_pb.LogEntry{
 		TsNs:             ts.UnixNano(),
 		PartitionKeyHash: util.HashToInt32(key),
@@ -58,14 +68,6 @@ func (m *LogBuffer) AddToBuffer(ts time.Time, key, data []byte) {
 
 	size := len(logEntryData)
 
-	m.Lock()
-	defer func() {
-		m.Unlock()
-		if m.notifyFn != nil {
-			m.notifyFn()
-		}
-	}()
-
 	if m.pos == 0 {
 		m.startTime = ts
 	}
@@ -153,18 +155,18 @@ func (m *LogBuffer) ReadFromBuffer(lastReadTime time.Time) (ts time.Time, buffer
 	l, h := 0, len(m.idx)-1
 
 	/*
-	for i, pos := range m.idx {
-		logEntry, ts := readTs(m.buf, pos)
-		event := &filer_pb.FullEventNotification{}
-		proto.Unmarshal(logEntry.Data, event)
-		entry := event.EventNotification.OldEntry
-		if entry == nil {
-			entry = event.EventNotification.NewEntry
+		for i, pos := range m.idx {
+			logEntry, ts := readTs(m.buf, pos)
+			event := &filer_pb.FullEventNotification{}
+			proto.Unmarshal(logEntry.Data, event)
+			entry := event.EventNotification.OldEntry
+			if entry == nil {
+				entry = event.EventNotification.NewEntry
+			}
+			fmt.Printf("entry %d ts: %v offset:%d dir:%s name:%s\n", i, time.Unix(0, ts), pos, event.Directory, entry.Name)
 		}
-		fmt.Printf("entry %d ts: %v offset:%d dir:%s name:%s\n", i, time.Unix(0, ts), pos, event.Directory, entry.Name)
-	}
-	fmt.Printf("l=%d, h=%d\n", l, h)
-	 */
+		fmt.Printf("l=%d, h=%d\n", l, h)
+	*/
 
 	for l <= h {
 		mid := (l + h) / 2

From 1101a42e5cb4f12d218023669ec57b8cbfb542e0 Mon Sep 17 00:00:00 2001
From: Chris Lu <chris.lu@gmail.com>
Date: Fri, 10 Apr 2020 01:37:03 -0700
Subject: [PATCH 26/81] mv : create new folder, move children, and delete old
 folder

---
 weed/server/filer_grpc_server_rename.go | 28 +++++++++++++++++++------
 1 file changed, 22 insertions(+), 6 deletions(-)

diff --git a/weed/server/filer_grpc_server_rename.go b/weed/server/filer_grpc_server_rename.go
index 306e4e5e0..c18587b45 100644
--- a/weed/server/filer_grpc_server_rename.go
+++ b/weed/server/filer_grpc_server_rename.go
@@ -44,12 +44,19 @@ func (fs *FilerServer) AtomicRenameEntry(ctx context.Context, req *filer_pb.Atom
 }
 
 func (fs *FilerServer) moveEntry(ctx context.Context, oldParent util.FullPath, entry *filer2.Entry, newParent util.FullPath, newName string, events *MoveEvents) error {
-	if entry.IsDirectory() {
-		if err := fs.moveFolderSubEntries(ctx, oldParent, entry, newParent, newName, events); err != nil {
-			return err
+
+	if err := fs.moveSelfEntry(ctx, oldParent, entry, newParent, newName, events, func() error {
+		if entry.IsDirectory() {
+			if err := fs.moveFolderSubEntries(ctx, oldParent, entry, newParent, newName, events); err != nil {
+				return err
+			}
 		}
+		return nil
+	}); err != nil {
+		return fmt.Errorf("fail to move %s => %s: %v", oldParent.Child(entry.Name()), newParent.Child(newName), err)
 	}
-	return fs.moveSelfEntry(ctx, oldParent, entry, newParent, newName, events)
+
+	return nil
 }
 
 func (fs *FilerServer) moveFolderSubEntries(ctx context.Context, oldParent util.FullPath, entry *filer2.Entry, newParent util.FullPath, newName string, events *MoveEvents) error {
@@ -85,7 +92,8 @@ func (fs *FilerServer) moveFolderSubEntries(ctx context.Context, oldParent util.
 	return nil
 }
 
-func (fs *FilerServer) moveSelfEntry(ctx context.Context, oldParent util.FullPath, entry *filer2.Entry, newParent util.FullPath, newName string, events *MoveEvents) error {
+func (fs *FilerServer) moveSelfEntry(ctx context.Context, oldParent util.FullPath, entry *filer2.Entry, newParent util.FullPath, newName string, events *MoveEvents,
+	moveFolderSubEntries func() error) error {
 
 	oldPath, newPath := oldParent.Child(entry.Name()), newParent.Child(newName)
 
@@ -107,6 +115,14 @@ func (fs *FilerServer) moveSelfEntry(ctx context.Context, oldParent util.FullPat
 		return createErr
 	}
 
+	events.newEntries = append(events.newEntries, newEntry)
+
+	if moveFolderSubEntries!=nil {
+		if moveChildrenErr := moveFolderSubEntries(); moveChildrenErr != nil {
+			return moveChildrenErr
+		}
+	}
+
 	// delete old entry
 	deleteErr := fs.filer.DeleteEntryMetaAndData(ctx, oldPath, false, false, false)
 	if deleteErr != nil {
@@ -114,7 +130,7 @@ func (fs *FilerServer) moveSelfEntry(ctx context.Context, oldParent util.FullPat
 	}
 
 	events.oldEntries = append(events.oldEntries, entry)
-	events.newEntries = append(events.newEntries, newEntry)
+
 	return nil
 
 }

From 13ab1fdaf0a4c67786d53760e8b9e6a81c35a04f Mon Sep 17 00:00:00 2001
From: Chris Lu <chris.lu@gmail.com>
Date: Fri, 10 Apr 2020 02:23:23 -0700
Subject: [PATCH 27/81] mount: use file default permission 0666

---
 weed/filesys/filehandle.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/weed/filesys/filehandle.go b/weed/filesys/filehandle.go
index 8b1be5209..935f0b314 100644
--- a/weed/filesys/filehandle.go
+++ b/weed/filesys/filehandle.go
@@ -193,7 +193,7 @@ func (fh *FileHandle) Flush(ctx context.Context, req *fuse.FlushRequest) error {
 			fh.f.entry.Attributes.Gid = req.Gid
 			fh.f.entry.Attributes.Mtime = time.Now().Unix()
 			fh.f.entry.Attributes.Crtime = time.Now().Unix()
-			fh.f.entry.Attributes.FileMode = uint32(0777 &^ fh.f.wfs.option.Umask)
+			fh.f.entry.Attributes.FileMode = uint32(0666 &^ fh.f.wfs.option.Umask)
 			fh.f.entry.Attributes.Collection = fh.dirtyPages.collection
 			fh.f.entry.Attributes.Replication = fh.dirtyPages.replication
 		}

From dce2702e1b599e2b1bef1bddb0495f96704d297f Mon Sep 17 00:00:00 2001
From: Chris Lu <chris.lu@gmail.com>
Date: Fri, 10 Apr 2020 14:50:10 -0700
Subject: [PATCH 28/81] fix the instruction

---
 weed/command/scaffold.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/weed/command/scaffold.go b/weed/command/scaffold.go
index 2aad29548..468476ae4 100644
--- a/weed/command/scaffold.go
+++ b/weed/command/scaffold.go
@@ -18,7 +18,7 @@ var cmdScaffold = &Command{
 	For example, the filer.toml mysql password can be overwritten by environment variable
 		export WEED_MYSQL_PASSWORD=some_password
 	Environment variable rules:
-		* Prefix fix with "WEED_"
+		* Prefix the variable name with "WEED_"
 		* Upppercase the reset of variable name.
 		* Replace '.' with '_'
 

From 4b27afe176645be853433e2fb6407c2e4324df2f Mon Sep 17 00:00:00 2001
From: Chris Lu <chris.lu@gmail.com>
Date: Fri, 10 Apr 2020 15:27:47 -0700
Subject: [PATCH 29/81] fix sqs message reading

---
 weed/replication/sub/notification_aws_sqs.go | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/weed/replication/sub/notification_aws_sqs.go b/weed/replication/sub/notification_aws_sqs.go
index 06869e619..1dd386ba7 100644
--- a/weed/replication/sub/notification_aws_sqs.go
+++ b/weed/replication/sub/notification_aws_sqs.go
@@ -92,7 +92,9 @@ func (k *AwsSqsInput) ReceiveMessage() (key string, message *filer_pb.EventNotif
 	}
 
 	// process the message
-	key = *result.Messages[0].Attributes["key"]
+	// fmt.Printf("messages: %+v\n", result.Messages[0])
+	keyValue := result.Messages[0].MessageAttributes["key"]
+	key = *keyValue.StringValue
 	text := *result.Messages[0].Body
 	message = &filer_pb.EventNotification{}
 	err = proto.UnmarshalText(text, message)

From 2aab4289ddbf826e42813724a0612c0912c561c0 Mon Sep 17 00:00:00 2001
From: Chris Lu <chrislusf@users.noreply.github.com>
Date: Fri, 10 Apr 2020 21:58:34 -0700
Subject: [PATCH 30/81] Update README.md

---
 README.md | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/README.md b/README.md
index f0d5412fc..90dc586fd 100644
--- a/README.md
+++ b/README.md
@@ -450,6 +450,12 @@ go get github.com/chrislusf/seaweedfs/weed
 
 Once this is done, you will find the executable "weed" in your `$GOPATH/bin` directory
 
+Note:
+* If you got into this problem, try to `rm -Rf $GOPATH/src/go.etcd.io/etcd/vendor` and build again.
+```
+panic: /debug/requests is already registered. You may have two independent copies of golang.org/x/net/trace in your binary, trying to maintain separate state. This may involve a vendored copy of golang.org/x/net/trace.
+```
+
 Step 4: after you modify your code locally, you could start a local build by calling `go install` under
 
 ```

From 7e7ed767d9aad09d254eb696eb00107171b17b7c Mon Sep 17 00:00:00 2001
From: Chris Lu <chrislusf@users.noreply.github.com>
Date: Fri, 10 Apr 2020 22:15:45 -0700
Subject: [PATCH 31/81] Update README.md

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 90dc586fd..8c63b3eb0 100644
--- a/README.md
+++ b/README.md
@@ -451,7 +451,7 @@ go get github.com/chrislusf/seaweedfs/weed
 Once this is done, you will find the executable "weed" in your `$GOPATH/bin` directory
 
 Note:
-* If you got into this problem, try to `rm -Rf $GOPATH/src/go.etcd.io/etcd/vendor` and build again.
+* If you got into this problem, try to `rm -Rf $GOPATH/src/go.etcd.io/etcd/vendor/golang.org/x/net/trace` and build again.
 ```
 panic: /debug/requests is already registered. You may have two independent copies of golang.org/x/net/trace in your binary, trying to maintain separate state. This may involve a vendored copy of golang.org/x/net/trace.
 ```

From 417125457e104b1447350b2be06808dd95d7b312 Mon Sep 17 00:00:00 2001
From: Chris Lu <chris.lu@gmail.com>
Date: Sat, 11 Apr 2020 02:19:45 -0700
Subject: [PATCH 32/81] add additional buffers for logs

---
 weed/queue/log_buffer.go    |  8 +++++++-
 weed/queue/sealed_buffer.go | 40 +++++++++++++++++++++++++++++++++++++
 2 files changed, 47 insertions(+), 1 deletion(-)
 create mode 100644 weed/queue/sealed_buffer.go

diff --git a/weed/queue/log_buffer.go b/weed/queue/log_buffer.go
index f24429479..78c5b75ae 100644
--- a/weed/queue/log_buffer.go
+++ b/weed/queue/log_buffer.go
@@ -11,6 +11,9 @@ import (
 	"github.com/chrislusf/seaweedfs/weed/util"
 )
 
+const BufferSize = 4 * 1024 * 1024
+const PreviousBufferCount = 3
+
 type dataToFlush struct {
 	startTime time.Time
 	stopTime  time.Time
@@ -18,6 +21,7 @@ type dataToFlush struct {
 }
 
 type LogBuffer struct {
+	prevBuffers   *SealedBuffers
 	buf           []byte
 	idx           []int
 	pos           int
@@ -34,7 +38,8 @@ type LogBuffer struct {
 
 func NewLogBuffer(flushInterval time.Duration, flushFn func(startTime, stopTime time.Time, buf []byte), notifyFn func()) *LogBuffer {
 	lb := &LogBuffer{
-		buf:           make([]byte, 4*1024*1024),
+		prevBuffers:   newSealedBuffers(PreviousBufferCount),
+		buf:           make([]byte, BufferSize),
 		sizeBuf:       make([]byte, 4),
 		flushInterval: flushInterval,
 		flushFn:       flushFn,
@@ -127,6 +132,7 @@ func (m *LogBuffer) copyToFlush() *dataToFlush {
 			stopTime:  m.stopTime,
 			data:      copiedBytes(m.buf[:m.pos]),
 		}
+		m.buf = m.prevBuffers.SealBuffer(m.startTime, m.stopTime, m.buf)
 		m.pos = 0
 		m.idx = m.idx[:0]
 		return d
diff --git a/weed/queue/sealed_buffer.go b/weed/queue/sealed_buffer.go
new file mode 100644
index 000000000..23cec92c9
--- /dev/null
+++ b/weed/queue/sealed_buffer.go
@@ -0,0 +1,40 @@
+package queue
+
+import "time"
+
+type MemBuffer struct {
+	buf       []byte
+	startTime time.Time
+	stopTime  time.Time
+}
+
+type SealedBuffers struct {
+	buffers []*MemBuffer
+}
+
+func newSealedBuffers(size int) *SealedBuffers {
+	sbs := &SealedBuffers{}
+
+	sbs.buffers = make([]*MemBuffer, size)
+	for i := 0; i < size; i++ {
+		sbs.buffers[i] = &MemBuffer{
+			buf: make([]byte, BufferSize),
+		}
+	}
+
+	return sbs
+}
+
+func (sbs *SealedBuffers) SealBuffer(startTime, stopTime time.Time, buf []byte) (newBuf []byte) {
+	oldMemBuffer := sbs.buffers[0]
+	size := len(sbs.buffers)
+	for i := 0; i < size-1; i++ {
+		sbs.buffers[i].buf = sbs.buffers[i+1].buf
+		sbs.buffers[i].startTime = sbs.buffers[i+1].startTime
+		sbs.buffers[i].stopTime = sbs.buffers[i+1].stopTime
+	}
+	sbs.buffers[size-1].buf = buf
+	sbs.buffers[size-1].startTime = startTime
+	sbs.buffers[size-1].stopTime = stopTime
+	return oldMemBuffer.buf
+}

From b7f0ba3800b2a2e9a1268e8b90714d3ab1281c72 Mon Sep 17 00:00:00 2001
From: Chris Lu <chris.lu@gmail.com>
Date: Sat, 11 Apr 2020 12:37:41 -0700
Subject: [PATCH 33/81] refactoring

---
 weed/filer2/filer.go                        | 6 +++---
 weed/{queue => log_buffer}/log_buffer.go    | 2 +-
 weed/{queue => log_buffer}/sealed_buffer.go | 2 +-
 3 files changed, 5 insertions(+), 5 deletions(-)
 rename weed/{queue => log_buffer}/log_buffer.go (99%)
 rename weed/{queue => log_buffer}/sealed_buffer.go (97%)

diff --git a/weed/filer2/filer.go b/weed/filer2/filer.go
index af17bf56c..578a2707f 100644
--- a/weed/filer2/filer.go
+++ b/weed/filer2/filer.go
@@ -12,8 +12,8 @@ import (
 	"github.com/karlseguin/ccache"
 
 	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/log_buffer"
 	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
-	"github.com/chrislusf/seaweedfs/weed/queue"
 	"github.com/chrislusf/seaweedfs/weed/util"
 	"github.com/chrislusf/seaweedfs/weed/wdclient"
 )
@@ -35,7 +35,7 @@ type Filer struct {
 	DirQueuesPath       string
 	buckets             *FilerBuckets
 	Cipher              bool
-	metaLogBuffer       *queue.LogBuffer
+	metaLogBuffer       *log_buffer.LogBuffer
 }
 
 func NewFiler(masters []string, grpcDialOption grpc.DialOption, filerGrpcPort uint32, notifyFn func()) *Filer {
@@ -45,7 +45,7 @@ func NewFiler(masters []string, grpcDialOption grpc.DialOption, filerGrpcPort ui
 		fileIdDeletionQueue: util.NewUnboundedQueue(),
 		GrpcDialOption:      grpcDialOption,
 	}
-	f.metaLogBuffer = queue.NewLogBuffer(time.Minute, f.logFlushFunc, notifyFn)
+	f.metaLogBuffer = log_buffer.NewLogBuffer(time.Minute, f.logFlushFunc, notifyFn)
 
 	go f.loopProcessingDeletion()
 
diff --git a/weed/queue/log_buffer.go b/weed/log_buffer/log_buffer.go
similarity index 99%
rename from weed/queue/log_buffer.go
rename to weed/log_buffer/log_buffer.go
index 78c5b75ae..3b3396128 100644
--- a/weed/queue/log_buffer.go
+++ b/weed/log_buffer/log_buffer.go
@@ -1,4 +1,4 @@
-package queue
+package log_buffer
 
 import (
 	"sync"
diff --git a/weed/queue/sealed_buffer.go b/weed/log_buffer/sealed_buffer.go
similarity index 97%
rename from weed/queue/sealed_buffer.go
rename to weed/log_buffer/sealed_buffer.go
index 23cec92c9..c5160fad0 100644
--- a/weed/queue/sealed_buffer.go
+++ b/weed/log_buffer/sealed_buffer.go
@@ -1,4 +1,4 @@
-package queue
+package log_buffer
 
 import "time"
 

From e909b55633de263b363359c3e045c39c67834ccf Mon Sep 17 00:00:00 2001
From: Chris Lu <chris.lu@gmail.com>
Date: Sat, 11 Apr 2020 12:39:46 -0700
Subject: [PATCH 34/81] refactoring

---
 weed/filer2/reader_at.go                  | 2 +-
 weed/filesys/wfs.go                       | 2 +-
 weed/server/webdav_server.go              | 2 +-
 weed/{pb => util}/pb_cache/chunk_cache.go | 0
 4 files changed, 3 insertions(+), 3 deletions(-)
 rename weed/{pb => util}/pb_cache/chunk_cache.go (100%)

diff --git a/weed/filer2/reader_at.go b/weed/filer2/reader_at.go
index b9913d2ca..448aa3465 100644
--- a/weed/filer2/reader_at.go
+++ b/weed/filer2/reader_at.go
@@ -9,8 +9,8 @@ import (
 
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
-	"github.com/chrislusf/seaweedfs/weed/pb/pb_cache"
 	"github.com/chrislusf/seaweedfs/weed/util"
+	"github.com/chrislusf/seaweedfs/weed/util/pb_cache"
 	"github.com/chrislusf/seaweedfs/weed/wdclient"
 )
 
diff --git a/weed/filesys/wfs.go b/weed/filesys/wfs.go
index 191b79845..5e994d14e 100644
--- a/weed/filesys/wfs.go
+++ b/weed/filesys/wfs.go
@@ -15,8 +15,8 @@ import (
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/pb"
 	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
-	"github.com/chrislusf/seaweedfs/weed/pb/pb_cache"
 	"github.com/chrislusf/seaweedfs/weed/util"
+	"github.com/chrislusf/seaweedfs/weed/util/pb_cache"
 	"github.com/seaweedfs/fuse"
 	"github.com/seaweedfs/fuse/fs"
 )
diff --git a/weed/server/webdav_server.go b/weed/server/webdav_server.go
index f4f3b44db..5a9861fe7 100644
--- a/weed/server/webdav_server.go
+++ b/weed/server/webdav_server.go
@@ -16,8 +16,8 @@ import (
 	"github.com/chrislusf/seaweedfs/weed/operation"
 	"github.com/chrislusf/seaweedfs/weed/pb"
 	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
-	"github.com/chrislusf/seaweedfs/weed/pb/pb_cache"
 	"github.com/chrislusf/seaweedfs/weed/util"
+	"github.com/chrislusf/seaweedfs/weed/util/pb_cache"
 
 	"github.com/chrislusf/seaweedfs/weed/filer2"
 	"github.com/chrislusf/seaweedfs/weed/glog"
diff --git a/weed/pb/pb_cache/chunk_cache.go b/weed/util/pb_cache/chunk_cache.go
similarity index 100%
rename from weed/pb/pb_cache/chunk_cache.go
rename to weed/util/pb_cache/chunk_cache.go

From ed5468259867502facd8df0e3e90574421abfc94 Mon Sep 17 00:00:00 2001
From: Chris Lu <chris.lu@gmail.com>
Date: Sat, 11 Apr 2020 12:43:17 -0700
Subject: [PATCH 35/81] refactoring

---
 weed/filer2/filer.go                        | 2 +-
 weed/{ => util}/log_buffer/log_buffer.go    | 0
 weed/{ => util}/log_buffer/sealed_buffer.go | 0
 3 files changed, 1 insertion(+), 1 deletion(-)
 rename weed/{ => util}/log_buffer/log_buffer.go (100%)
 rename weed/{ => util}/log_buffer/sealed_buffer.go (100%)

diff --git a/weed/filer2/filer.go b/weed/filer2/filer.go
index 578a2707f..dde08a092 100644
--- a/weed/filer2/filer.go
+++ b/weed/filer2/filer.go
@@ -12,9 +12,9 @@ import (
 	"github.com/karlseguin/ccache"
 
 	"github.com/chrislusf/seaweedfs/weed/glog"
-	"github.com/chrislusf/seaweedfs/weed/log_buffer"
 	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
 	"github.com/chrislusf/seaweedfs/weed/util"
+	"github.com/chrislusf/seaweedfs/weed/util/log_buffer"
 	"github.com/chrislusf/seaweedfs/weed/wdclient"
 )
 
diff --git a/weed/log_buffer/log_buffer.go b/weed/util/log_buffer/log_buffer.go
similarity index 100%
rename from weed/log_buffer/log_buffer.go
rename to weed/util/log_buffer/log_buffer.go
diff --git a/weed/log_buffer/sealed_buffer.go b/weed/util/log_buffer/sealed_buffer.go
similarity index 100%
rename from weed/log_buffer/sealed_buffer.go
rename to weed/util/log_buffer/sealed_buffer.go

From d7f3acb2c056534f29950f3586d804ec274349b2 Mon Sep 17 00:00:00 2001
From: Chris Lu <chris.lu@gmail.com>
Date: Sat, 11 Apr 2020 12:45:24 -0700
Subject: [PATCH 36/81] refactor

---
 weed/filer2/reader_at.go                           | 6 +++---
 weed/filesys/wfs.go                                | 6 +++---
 weed/server/webdav_server.go                       | 6 +++---
 weed/util/{pb_cache => chunk_cache}/chunk_cache.go | 2 +-
 4 files changed, 10 insertions(+), 10 deletions(-)
 rename weed/util/{pb_cache => chunk_cache}/chunk_cache.go (97%)

diff --git a/weed/filer2/reader_at.go b/weed/filer2/reader_at.go
index 448aa3465..2782cc204 100644
--- a/weed/filer2/reader_at.go
+++ b/weed/filer2/reader_at.go
@@ -10,7 +10,7 @@ import (
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
 	"github.com/chrislusf/seaweedfs/weed/util"
-	"github.com/chrislusf/seaweedfs/weed/util/pb_cache"
+	"github.com/chrislusf/seaweedfs/weed/util/chunk_cache"
 	"github.com/chrislusf/seaweedfs/weed/wdclient"
 )
 
@@ -22,12 +22,12 @@ type ChunkReadAt struct {
 	lookupFileId func(fileId string) (targetUrl string, err error)
 	readerLock   sync.Mutex
 
-	chunkCache *pb_cache.ChunkCache
+	chunkCache *chunk_cache.ChunkCache
 }
 
 // var _ = io.ReaderAt(&ChunkReadAt{})
 
-func NewChunkReaderAtFromClient(filerClient filer_pb.FilerClient, chunkViews []*ChunkView, chunkCache *pb_cache.ChunkCache) *ChunkReadAt {
+func NewChunkReaderAtFromClient(filerClient filer_pb.FilerClient, chunkViews []*ChunkView, chunkCache *chunk_cache.ChunkCache) *ChunkReadAt {
 
 	return &ChunkReadAt{
 		chunkViews: chunkViews,
diff --git a/weed/filesys/wfs.go b/weed/filesys/wfs.go
index 5e994d14e..49db18b6e 100644
--- a/weed/filesys/wfs.go
+++ b/weed/filesys/wfs.go
@@ -16,7 +16,7 @@ import (
 	"github.com/chrislusf/seaweedfs/weed/pb"
 	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
 	"github.com/chrislusf/seaweedfs/weed/util"
-	"github.com/chrislusf/seaweedfs/weed/util/pb_cache"
+	"github.com/chrislusf/seaweedfs/weed/util/chunk_cache"
 	"github.com/seaweedfs/fuse"
 	"github.com/seaweedfs/fuse/fs"
 )
@@ -64,7 +64,7 @@ type WFS struct {
 	root        fs.Node
 	fsNodeCache *FsCache
 
-	chunkCache *pb_cache.ChunkCache
+	chunkCache *chunk_cache.ChunkCache
 }
 type statsCache struct {
 	filer_pb.StatisticsResponse
@@ -81,7 +81,7 @@ func NewSeaweedFileSystem(option *Option) *WFS {
 				return make([]byte, option.ChunkSizeLimit)
 			},
 		},
-		chunkCache: pb_cache.NewChunkCache(option.ChunkCacheCountLimit),
+		chunkCache: chunk_cache.NewChunkCache(option.ChunkCacheCountLimit),
 	}
 
 	wfs.root = &Dir{name: wfs.option.FilerMountRootPath, wfs: wfs}
diff --git a/weed/server/webdav_server.go b/weed/server/webdav_server.go
index 5a9861fe7..082755291 100644
--- a/weed/server/webdav_server.go
+++ b/weed/server/webdav_server.go
@@ -17,7 +17,7 @@ import (
 	"github.com/chrislusf/seaweedfs/weed/pb"
 	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
 	"github.com/chrislusf/seaweedfs/weed/util"
-	"github.com/chrislusf/seaweedfs/weed/util/pb_cache"
+	"github.com/chrislusf/seaweedfs/weed/util/chunk_cache"
 
 	"github.com/chrislusf/seaweedfs/weed/filer2"
 	"github.com/chrislusf/seaweedfs/weed/glog"
@@ -67,7 +67,7 @@ type WebDavFileSystem struct {
 	secret         security.SigningKey
 	filer          *filer2.Filer
 	grpcDialOption grpc.DialOption
-	chunkCache     *pb_cache.ChunkCache
+	chunkCache     *chunk_cache.ChunkCache
 }
 
 type FileInfo struct {
@@ -98,7 +98,7 @@ type WebDavFile struct {
 func NewWebDavFileSystem(option *WebDavOption) (webdav.FileSystem, error) {
 	return &WebDavFileSystem{
 		option:     option,
-		chunkCache: pb_cache.NewChunkCache(1000),
+		chunkCache: chunk_cache.NewChunkCache(1000),
 	}, nil
 }
 
diff --git a/weed/util/pb_cache/chunk_cache.go b/weed/util/chunk_cache/chunk_cache.go
similarity index 97%
rename from weed/util/pb_cache/chunk_cache.go
rename to weed/util/chunk_cache/chunk_cache.go
index d729bd8c1..e2676d9cc 100644
--- a/weed/util/pb_cache/chunk_cache.go
+++ b/weed/util/chunk_cache/chunk_cache.go
@@ -1,4 +1,4 @@
-package pb_cache
+package chunk_cache
 
 import (
 	"time"

From df9d538044c035edde5441972dffdc7662fe753b Mon Sep 17 00:00:00 2001
From: Chris Lu <chris.lu@gmail.com>
Date: Sat, 11 Apr 2020 14:19:44 -0700
Subject: [PATCH 37/81] rename function

---
 weed/storage/volume_create.go         | 2 +-
 weed/storage/volume_create_linux.go   | 2 +-
 weed/storage/volume_create_windows.go | 2 +-
 weed/storage/volume_loading.go        | 2 +-
 weed/storage/volume_vacuum.go         | 4 ++--
 5 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/weed/storage/volume_create.go b/weed/storage/volume_create.go
index ffcb246a4..422537a24 100644
--- a/weed/storage/volume_create.go
+++ b/weed/storage/volume_create.go
@@ -9,7 +9,7 @@ import (
 	"github.com/chrislusf/seaweedfs/weed/storage/backend"
 )
 
-func createVolumeFile(fileName string, preallocate int64, memoryMapSizeMB uint32) (backend.BackendStorageFile, error) {
+func CreateVolumeFile(fileName string, preallocate int64, memoryMapSizeMB uint32) (backend.BackendStorageFile, error) {
 	file, e := os.OpenFile(fileName, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644)
 	if e != nil {
 		return nil, e
diff --git a/weed/storage/volume_create_linux.go b/weed/storage/volume_create_linux.go
index ee599ac32..a854c531a 100644
--- a/weed/storage/volume_create_linux.go
+++ b/weed/storage/volume_create_linux.go
@@ -10,7 +10,7 @@ import (
 	"github.com/chrislusf/seaweedfs/weed/storage/backend"
 )
 
-func createVolumeFile(fileName string, preallocate int64, memoryMapSizeMB uint32) (backend.BackendStorageFile, error) {
+func CreateVolumeFile(fileName string, preallocate int64, memoryMapSizeMB uint32) (backend.BackendStorageFile, error) {
 	file, e := os.OpenFile(fileName, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644)
 	if e != nil {
 		return nil, e
diff --git a/weed/storage/volume_create_windows.go b/weed/storage/volume_create_windows.go
index e1c0b961f..e347cbd1c 100644
--- a/weed/storage/volume_create_windows.go
+++ b/weed/storage/volume_create_windows.go
@@ -11,7 +11,7 @@ import (
 	"github.com/chrislusf/seaweedfs/weed/storage/backend/memory_map/os_overloads"
 )
 
-func createVolumeFile(fileName string, preallocate int64, memoryMapSizeMB uint32) (backend.BackendStorageFile, error) {
+func CreateVolumeFile(fileName string, preallocate int64, memoryMapSizeMB uint32) (backend.BackendStorageFile, error) {
 	if preallocate > 0 {
 		glog.V(0).Infof("Preallocated disk space for %s is not supported", fileName)
 	}
diff --git a/weed/storage/volume_loading.go b/weed/storage/volume_loading.go
index 3b0897bca..64979cba1 100644
--- a/weed/storage/volume_loading.go
+++ b/weed/storage/volume_loading.go
@@ -54,7 +54,7 @@ func (v *Volume) load(alsoLoadIndex bool, createDatIfMissing bool, needleMapKind
 		v.DataBackend = backend.NewDiskFile(dataFile)
 	} else {
 		if createDatIfMissing {
-			v.DataBackend, err = createVolumeFile(fileName+".dat", preallocate, v.MemoryMapMaxSizeMb)
+			v.DataBackend, err = CreateVolumeFile(fileName+".dat", preallocate, v.MemoryMapMaxSizeMb)
 		} else {
 			return fmt.Errorf("Volume Data file %s.dat does not exist.", fileName)
 		}
diff --git a/weed/storage/volume_vacuum.go b/weed/storage/volume_vacuum.go
index 67c3957de..ce3ab3a46 100644
--- a/weed/storage/volume_vacuum.go
+++ b/weed/storage/volume_vacuum.go
@@ -354,7 +354,7 @@ func (v *Volume) copyDataAndGenerateIndexFile(dstName, idxName string, prealloca
 	var (
 		dst backend.BackendStorageFile
 	)
-	if dst, err = createVolumeFile(dstName, preallocate, 0); err != nil {
+	if dst, err = CreateVolumeFile(dstName, preallocate, 0); err != nil {
 		return
 	}
 	defer dst.Close()
@@ -383,7 +383,7 @@ func copyDataBasedOnIndexFile(srcDatName, srcIdxName, dstDatName, datIdxName str
 		srcDatBackend, dstDatBackend backend.BackendStorageFile
 		dataFile                     *os.File
 	)
-	if dstDatBackend, err = createVolumeFile(dstDatName, preallocate, 0); err != nil {
+	if dstDatBackend, err = CreateVolumeFile(dstDatName, preallocate, 0); err != nil {
 		return
 	}
 	defer dstDatBackend.Close()

From c8ca234773e2a0c57c503c1f3464d1ded4edd2df Mon Sep 17 00:00:00 2001
From: Chris Lu <chris.lu@gmail.com>
Date: Sat, 11 Apr 2020 14:27:25 -0700
Subject: [PATCH 38/81] refactoring

---
 weed/storage/backend/memory_map/memory_map_backend.go | 4 +---
 weed/storage/{ => backend}/volume_create.go           | 7 +++----
 weed/storage/{ => backend}/volume_create_linux.go     | 7 +++----
 weed/storage/{ => backend}/volume_create_windows.go   | 7 +++----
 weed/storage/volume_loading.go                        | 2 +-
 weed/storage/volume_vacuum.go                         | 4 ++--
 6 files changed, 13 insertions(+), 18 deletions(-)
 rename weed/storage/{ => backend}/volume_create.go (68%)
 rename weed/storage/{ => backend}/volume_create_linux.go (72%)
 rename weed/storage/{ => backend}/volume_create_windows.go (82%)

diff --git a/weed/storage/backend/memory_map/memory_map_backend.go b/weed/storage/backend/memory_map/memory_map_backend.go
index 44ef4d3e1..7fc2b4920 100644
--- a/weed/storage/backend/memory_map/memory_map_backend.go
+++ b/weed/storage/backend/memory_map/memory_map_backend.go
@@ -3,12 +3,10 @@ package memory_map
 import (
 	"os"
 	"time"
-
-	"github.com/chrislusf/seaweedfs/weed/storage/backend"
 )
 
 var (
-	_ backend.BackendStorageFile = &MemoryMappedFile{}
+	// _ backend.BackendStorageFile = &MemoryMappedFile{} // remove this to break import cycle
 )
 
 type MemoryMappedFile struct {
diff --git a/weed/storage/volume_create.go b/weed/storage/backend/volume_create.go
similarity index 68%
rename from weed/storage/volume_create.go
rename to weed/storage/backend/volume_create.go
index 422537a24..abb1f7238 100644
--- a/weed/storage/volume_create.go
+++ b/weed/storage/backend/volume_create.go
@@ -1,15 +1,14 @@
 // +build !linux,!windows
 
-package storage
+package backend
 
 import (
 	"os"
 
 	"github.com/chrislusf/seaweedfs/weed/glog"
-	"github.com/chrislusf/seaweedfs/weed/storage/backend"
 )
 
-func CreateVolumeFile(fileName string, preallocate int64, memoryMapSizeMB uint32) (backend.BackendStorageFile, error) {
+func CreateVolumeFile(fileName string, preallocate int64, memoryMapSizeMB uint32) (BackendStorageFile, error) {
 	file, e := os.OpenFile(fileName, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644)
 	if e != nil {
 		return nil, e
@@ -17,5 +16,5 @@ func CreateVolumeFile(fileName string, preallocate int64, memoryMapSizeMB uint32
 	if preallocate > 0 {
 		glog.V(0).Infof("Preallocated disk space for %s is not supported", fileName)
 	}
-	return backend.NewDiskFile(file), nil
+	return NewDiskFile(file), nil
 }
diff --git a/weed/storage/volume_create_linux.go b/weed/storage/backend/volume_create_linux.go
similarity index 72%
rename from weed/storage/volume_create_linux.go
rename to weed/storage/backend/volume_create_linux.go
index a854c531a..4602831ca 100644
--- a/weed/storage/volume_create_linux.go
+++ b/weed/storage/backend/volume_create_linux.go
@@ -1,16 +1,15 @@
 // +build linux
 
-package storage
+package backend
 
 import (
 	"os"
 	"syscall"
 
 	"github.com/chrislusf/seaweedfs/weed/glog"
-	"github.com/chrislusf/seaweedfs/weed/storage/backend"
 )
 
-func CreateVolumeFile(fileName string, preallocate int64, memoryMapSizeMB uint32) (backend.BackendStorageFile, error) {
+func CreateVolumeFile(fileName string, preallocate int64, memoryMapSizeMB uint32) (BackendStorageFile, error) {
 	file, e := os.OpenFile(fileName, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644)
 	if e != nil {
 		return nil, e
@@ -19,5 +18,5 @@ func CreateVolumeFile(fileName string, preallocate int64, memoryMapSizeMB uint32
 		syscall.Fallocate(int(file.Fd()), 1, 0, preallocate)
 		glog.V(0).Infof("Preallocated %d bytes disk space for %s", preallocate, fileName)
 	}
-	return backend.NewDiskFile(file), nil
+	return NewDiskFile(file), nil
 }
diff --git a/weed/storage/volume_create_windows.go b/weed/storage/backend/volume_create_windows.go
similarity index 82%
rename from weed/storage/volume_create_windows.go
rename to weed/storage/backend/volume_create_windows.go
index e347cbd1c..7d40ec0d7 100644
--- a/weed/storage/volume_create_windows.go
+++ b/weed/storage/backend/volume_create_windows.go
@@ -1,17 +1,16 @@
 // +build windows
 
-package storage
+package backend
 
 import (
 	"github.com/chrislusf/seaweedfs/weed/storage/backend/memory_map"
 	"golang.org/x/sys/windows"
 
 	"github.com/chrislusf/seaweedfs/weed/glog"
-	"github.com/chrislusf/seaweedfs/weed/storage/backend"
 	"github.com/chrislusf/seaweedfs/weed/storage/backend/memory_map/os_overloads"
 )
 
-func CreateVolumeFile(fileName string, preallocate int64, memoryMapSizeMB uint32) (backend.BackendStorageFile, error) {
+func CreateVolumeFile(fileName string, preallocate int64, memoryMapSizeMB uint32) (BackendStorageFile, error) {
 	if preallocate > 0 {
 		glog.V(0).Infof("Preallocated disk space for %s is not supported", fileName)
 	}
@@ -27,7 +26,7 @@ func CreateVolumeFile(fileName string, preallocate int64, memoryMapSizeMB uint32
 		if e != nil {
 			return nil, e
 		}
-		return backend.NewDiskFile(file), nil
+		return NewDiskFile(file), nil
 	}
 
 }
diff --git a/weed/storage/volume_loading.go b/weed/storage/volume_loading.go
index 64979cba1..b0b17af75 100644
--- a/weed/storage/volume_loading.go
+++ b/weed/storage/volume_loading.go
@@ -54,7 +54,7 @@ func (v *Volume) load(alsoLoadIndex bool, createDatIfMissing bool, needleMapKind
 		v.DataBackend = backend.NewDiskFile(dataFile)
 	} else {
 		if createDatIfMissing {
-			v.DataBackend, err = CreateVolumeFile(fileName+".dat", preallocate, v.MemoryMapMaxSizeMb)
+			v.DataBackend, err = backend.CreateVolumeFile(fileName+".dat", preallocate, v.MemoryMapMaxSizeMb)
 		} else {
 			return fmt.Errorf("Volume Data file %s.dat does not exist.", fileName)
 		}
diff --git a/weed/storage/volume_vacuum.go b/weed/storage/volume_vacuum.go
index ce3ab3a46..7518240d2 100644
--- a/weed/storage/volume_vacuum.go
+++ b/weed/storage/volume_vacuum.go
@@ -354,7 +354,7 @@ func (v *Volume) copyDataAndGenerateIndexFile(dstName, idxName string, prealloca
 	var (
 		dst backend.BackendStorageFile
 	)
-	if dst, err = CreateVolumeFile(dstName, preallocate, 0); err != nil {
+	if dst, err = backend.CreateVolumeFile(dstName, preallocate, 0); err != nil {
 		return
 	}
 	defer dst.Close()
@@ -383,7 +383,7 @@ func copyDataBasedOnIndexFile(srcDatName, srcIdxName, dstDatName, datIdxName str
 		srcDatBackend, dstDatBackend backend.BackendStorageFile
 		dataFile                     *os.File
 	)
-	if dstDatBackend, err = CreateVolumeFile(dstDatName, preallocate, 0); err != nil {
+	if dstDatBackend, err = backend.CreateVolumeFile(dstDatName, preallocate, 0); err != nil {
 		return
 	}
 	defer dstDatBackend.Close()

From df97da25f902912dd527d4aed567408c3ca0f9ae Mon Sep 17 00:00:00 2001
From: Chris Lu <chris.lu@gmail.com>
Date: Sat, 11 Apr 2020 21:12:41 -0700
Subject: [PATCH 39/81] mount: add on disk caching

---
 weed/command/mount.go                         |  10 +-
 weed/command/mount_std.go                     |   6 +-
 weed/command/webdav.go                        |   7 +
 weed/filesys/wfs.go                           |  31 ++--
 weed/server/webdav_server.go                  |   9 +-
 weed/util/chunk_cache/chunk_cache.go          | 111 ++++++++++++--
 .../util/chunk_cache/chunk_cache_in_memory.go |  36 +++++
 weed/util/chunk_cache/chunk_cache_on_disk.go  | 145 ++++++++++++++++++
 .../chunk_cache/chunk_cache_on_disk_test.go   |  58 +++++++
 9 files changed, 378 insertions(+), 35 deletions(-)
 create mode 100644 weed/util/chunk_cache/chunk_cache_in_memory.go
 create mode 100644 weed/util/chunk_cache/chunk_cache_on_disk.go
 create mode 100644 weed/util/chunk_cache/chunk_cache_on_disk_test.go

diff --git a/weed/command/mount.go b/weed/command/mount.go
index adf384a6f..6165402b4 100644
--- a/weed/command/mount.go
+++ b/weed/command/mount.go
@@ -1,5 +1,9 @@
 package command
 
+import (
+	"os"
+)
+
 type MountOptions struct {
 	filer                       *string
 	filerMountRootPath          *string
@@ -9,7 +13,8 @@ type MountOptions struct {
 	replication                 *string
 	ttlSec                      *int
 	chunkSizeLimitMB            *int
-	chunkCacheCountLimit        *int64
+	cacheDir                    *string
+	cacheSizeMB                 *int64
 	dataCenter                  *string
 	allowOthers                 *bool
 	umaskString                 *string
@@ -33,7 +38,8 @@ func init() {
 	mountOptions.replication = cmdMount.Flag.String("replication", "", "replication(e.g. 000, 001) to create to files. If empty, let filer decide.")
 	mountOptions.ttlSec = cmdMount.Flag.Int("ttl", 0, "file ttl in seconds")
 	mountOptions.chunkSizeLimitMB = cmdMount.Flag.Int("chunkSizeLimitMB", 4, "local write buffer size, also chunk large files")
-	mountOptions.chunkCacheCountLimit = cmdMount.Flag.Int64("chunkCacheCountLimit", 1000, "number of file chunks to cache in memory")
+	mountOptions.cacheDir = cmdMount.Flag.String("cacheDir", os.TempDir(), "local cache directory for file chunks")
+	mountOptions.cacheSizeMB = cmdMount.Flag.Int64("cacheCapacityMB", 1000, "local cache capacity in MB")
 	mountOptions.dataCenter = cmdMount.Flag.String("dataCenter", "", "prefer to write to the data center")
 	mountOptions.allowOthers = cmdMount.Flag.Bool("allowOthers", true, "allows other users to access the file system")
 	mountOptions.umaskString = cmdMount.Flag.String("umask", "022", "octal umask, e.g., 022, 0111")
diff --git a/weed/command/mount_std.go b/weed/command/mount_std.go
index 148540dec..0f87d6aee 100644
--- a/weed/command/mount_std.go
+++ b/weed/command/mount_std.go
@@ -129,7 +129,6 @@ func RunMount(option *MountOptions, umask os.FileMode) bool {
 	}
 
 	options = append(options, osSpecificMountOptions()...)
-
 	if *option.allowOthers {
 		options = append(options, fuse.AllowOther())
 	}
@@ -137,12 +136,12 @@ func RunMount(option *MountOptions, umask os.FileMode) bool {
 		options = append(options, fuse.AllowNonEmptyMount())
 	}
 
+	// mount
 	c, err := fuse.Mount(dir, options...)
 	if err != nil {
 		glog.V(0).Infof("mount: %v", err)
 		return true
 	}
-
 	defer fuse.Unmount(dir)
 
 	util.OnInterrupt(func() {
@@ -164,7 +163,8 @@ func RunMount(option *MountOptions, umask os.FileMode) bool {
 		Replication:                 *option.replication,
 		TtlSec:                      int32(*option.ttlSec),
 		ChunkSizeLimit:              int64(chunkSizeLimitMB) * 1024 * 1024,
-		ChunkCacheCountLimit:        *option.chunkCacheCountLimit,
+		CacheDir:                    *option.cacheDir,
+		CacheSizeMB:                 *option.cacheSizeMB,
 		DataCenter:                  *option.dataCenter,
 		DirListCacheLimit:           *option.dirListCacheLimit,
 		EntryCacheTtl:               3 * time.Second,
diff --git a/weed/command/webdav.go b/weed/command/webdav.go
index 4f5d5f5ce..a1616d0fc 100644
--- a/weed/command/webdav.go
+++ b/weed/command/webdav.go
@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"net/http"
+	"os"
 	"os/user"
 	"strconv"
 	"time"
@@ -26,6 +27,8 @@ type WebDavOption struct {
 	collection     *string
 	tlsPrivateKey  *string
 	tlsCertificate *string
+	cacheDir       *string
+	cacheSizeMB    *int64
 }
 
 func init() {
@@ -35,6 +38,8 @@ func init() {
 	webDavStandaloneOptions.collection = cmdWebDav.Flag.String("collection", "", "collection to create the files")
 	webDavStandaloneOptions.tlsPrivateKey = cmdWebDav.Flag.String("key.file", "", "path to the TLS private key file")
 	webDavStandaloneOptions.tlsCertificate = cmdWebDav.Flag.String("cert.file", "", "path to the TLS certificate file")
+	webDavStandaloneOptions.cacheDir = cmdWebDav.Flag.String("cacheDir", os.TempDir(), "local cache directory for file chunks")
+	webDavStandaloneOptions.cacheSizeMB = cmdWebDav.Flag.Int64("cacheCapacityMB", 1000, "local cache capacity in MB")
 }
 
 var cmdWebDav = &Command{
@@ -105,6 +110,8 @@ func (wo *WebDavOption) startWebDav() bool {
 		Uid:              uid,
 		Gid:              gid,
 		Cipher:           cipher,
+		CacheDir:         *wo.cacheDir,
+		CacheSizeMB:      *wo.cacheSizeMB,
 	})
 	if webdavServer_err != nil {
 		glog.Fatalf("WebDav Server startup error: %v", webdavServer_err)
diff --git a/weed/filesys/wfs.go b/weed/filesys/wfs.go
index 49db18b6e..b2f68c030 100644
--- a/weed/filesys/wfs.go
+++ b/weed/filesys/wfs.go
@@ -22,18 +22,19 @@ import (
 )
 
 type Option struct {
-	FilerGrpcAddress     string
-	GrpcDialOption       grpc.DialOption
-	FilerMountRootPath   string
-	Collection           string
-	Replication          string
-	TtlSec               int32
-	ChunkSizeLimit       int64
-	ChunkCacheCountLimit int64
-	DataCenter           string
-	DirListCacheLimit    int64
-	EntryCacheTtl        time.Duration
-	Umask                os.FileMode
+	FilerGrpcAddress   string
+	GrpcDialOption     grpc.DialOption
+	FilerMountRootPath string
+	Collection         string
+	Replication        string
+	TtlSec             int32
+	ChunkSizeLimit     int64
+	CacheDir           string
+	CacheSizeMB        int64
+	DataCenter         string
+	DirListCacheLimit  int64
+	EntryCacheTtl      time.Duration
+	Umask              os.FileMode
 
 	MountUid   uint32
 	MountGid   uint32
@@ -72,6 +73,10 @@ type statsCache struct {
 }
 
 func NewSeaweedFileSystem(option *Option) *WFS {
+	chunkCache := chunk_cache.NewChunkCache(256, option.CacheDir, option.CacheSizeMB, 4)
+	util.OnInterrupt(func() {
+		chunkCache.Shutdown()
+	})
 	wfs := &WFS{
 		option:                    option,
 		listDirectoryEntriesCache: ccache.New(ccache.Configure().MaxSize(option.DirListCacheLimit * 3).ItemsToPrune(100)),
@@ -81,7 +86,7 @@ func NewSeaweedFileSystem(option *Option) *WFS {
 				return make([]byte, option.ChunkSizeLimit)
 			},
 		},
-		chunkCache: chunk_cache.NewChunkCache(option.ChunkCacheCountLimit),
+		chunkCache: chunkCache,
 	}
 
 	wfs.root = &Dir{name: wfs.option.FilerMountRootPath, wfs: wfs}
diff --git a/weed/server/webdav_server.go b/weed/server/webdav_server.go
index 082755291..affc953bc 100644
--- a/weed/server/webdav_server.go
+++ b/weed/server/webdav_server.go
@@ -34,6 +34,8 @@ type WebDavOption struct {
 	Uid              uint32
 	Gid              uint32
 	Cipher           bool
+	CacheDir         string
+	CacheSizeMB      int64
 }
 
 type WebDavServer struct {
@@ -96,9 +98,14 @@ type WebDavFile struct {
 }
 
 func NewWebDavFileSystem(option *WebDavOption) (webdav.FileSystem, error) {
+
+	chunkCache := chunk_cache.NewChunkCache(256, option.CacheDir, option.CacheSizeMB, 4)
+	util.OnInterrupt(func() {
+		chunkCache.Shutdown()
+	})
 	return &WebDavFileSystem{
 		option:     option,
-		chunkCache: chunk_cache.NewChunkCache(1000),
+		chunkCache: chunkCache,
 	}, nil
 }
 
diff --git a/weed/util/chunk_cache/chunk_cache.go b/weed/util/chunk_cache/chunk_cache.go
index e2676d9cc..682f5185a 100644
--- a/weed/util/chunk_cache/chunk_cache.go
+++ b/weed/util/chunk_cache/chunk_cache.go
@@ -1,36 +1,115 @@
 package chunk_cache
 
 import (
-	"time"
+	"fmt"
+	"path"
+	"sort"
+	"sync"
 
-	"github.com/karlseguin/ccache"
+	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/storage"
+	"github.com/chrislusf/seaweedfs/weed/storage/needle"
 )
 
 // a global cache for recently accessed file chunks
 type ChunkCache struct {
-	cache *ccache.Cache
+	memCache   *ChunkCacheInMemory
+	diskCaches []*ChunkCacheVolume
+	sync.RWMutex
 }
 
-func NewChunkCache(maxEntries int64) *ChunkCache {
-	pruneCount := maxEntries >> 3
-	if pruneCount <= 0 {
-		pruneCount = 500
+func NewChunkCache(maxEntries int64, dir string, diskSizeMB int64, segmentCount int) *ChunkCache {
+	c := &ChunkCache{
+		memCache: NewChunkCacheInMemory(maxEntries),
 	}
-	return &ChunkCache{
-		cache: ccache.New(ccache.Configure().MaxSize(maxEntries).ItemsToPrune(uint32(pruneCount))),
+
+	volumeCount, volumeSize := int(diskSizeMB/30000), int64(30000)
+	if volumeCount < segmentCount {
+		volumeCount, volumeSize = segmentCount, diskSizeMB/int64(segmentCount)
+	}
+
+	for i := 0; i < volumeCount; i++ {
+		fileName := path.Join(dir, fmt.Sprintf("cache_%d", i))
+		diskCache, err := LoadOrCreateChunkCacheVolume(fileName, volumeSize*1024*1024)
+		if err != nil {
+			glog.Errorf("failed to add cache %s : %v", fileName, err)
+		} else {
+			c.diskCaches = append(c.diskCaches, diskCache)
+		}
 	}
+
+	// keep newest cache to the front
+	sort.Slice(c.diskCaches, func(i, j int) bool {
+		return c.diskCaches[i].lastModTime.After(c.diskCaches[j].lastModTime)
+	})
+
+	return c
 }
 
-func (c *ChunkCache) GetChunk(fileId string) []byte {
-	item := c.cache.Get(fileId)
-	if item == nil {
+func (c *ChunkCache) GetChunk(fileId string) (data []byte) {
+	c.RLock()
+	defer c.RUnlock()
+
+	if data = c.memCache.GetChunk(fileId); data != nil {
+		return data
+	}
+
+	fid, err := needle.ParseFileIdFromString(fileId)
+	if err != nil {
+		glog.Errorf("failed to parse file id %s", fileId)
 		return nil
 	}
-	data := item.Value().([]byte)
-	item.Extend(time.Hour)
-	return data
+	for _, diskCache := range c.diskCaches {
+		data, err = diskCache.GetNeedle(fid.Key)
+		if err == storage.ErrorNotFound {
+			continue
+		}
+		if err != nil {
+			glog.Errorf("failed to read cache file %s id %s", diskCache.fileName, fileId)
+			continue
+		}
+		if len(data) != 0 {
+			return
+		}
+	}
+	return nil
 }
 
 func (c *ChunkCache) SetChunk(fileId string, data []byte) {
-	c.cache.Set(fileId, data, time.Hour)
+	c.Lock()
+	defer c.Unlock()
+
+	c.memCache.SetChunk(fileId, data)
+
+	if len(c.diskCaches) == 0 {
+		return
+	}
+
+	if c.diskCaches[0].fileSize+int64(len(data)) > c.diskCaches[0].sizeLimit {
+		t, resetErr := c.diskCaches[len(c.diskCaches)-1].Reset()
+		if resetErr != nil {
+			glog.Errorf("failed to reset cache file %s", c.diskCaches[len(c.diskCaches)-1].fileName)
+			return
+		}
+		for i := len(c.diskCaches) - 1; i > 0; i-- {
+			c.diskCaches[i] = c.diskCaches[i-1]
+		}
+		c.diskCaches[0] = t
+	}
+
+	fid, err := needle.ParseFileIdFromString(fileId)
+	if err != nil {
+		glog.Errorf("failed to parse file id %s", fileId)
+		return
+	}
+	c.diskCaches[0].WriteNeedle(fid.Key, data)
+
 }
+
+func (c *ChunkCache) Shutdown() {
+	c.Lock()
+	defer c.Unlock()
+	for _, diskCache := range c.diskCaches {
+		diskCache.Shutdown()
+	}
+}
\ No newline at end of file
diff --git a/weed/util/chunk_cache/chunk_cache_in_memory.go b/weed/util/chunk_cache/chunk_cache_in_memory.go
new file mode 100644
index 000000000..931e45e9a
--- /dev/null
+++ b/weed/util/chunk_cache/chunk_cache_in_memory.go
@@ -0,0 +1,36 @@
+package chunk_cache
+
+import (
+	"time"
+
+	"github.com/karlseguin/ccache"
+)
+
+// a global cache for recently accessed file chunks
+type ChunkCacheInMemory struct {
+	cache *ccache.Cache
+}
+
+func NewChunkCacheInMemory(maxEntries int64) *ChunkCacheInMemory {
+	pruneCount := maxEntries >> 3
+	if pruneCount <= 0 {
+		pruneCount = 500
+	}
+	return &ChunkCacheInMemory{
+		cache: ccache.New(ccache.Configure().MaxSize(maxEntries).ItemsToPrune(uint32(pruneCount))),
+	}
+}
+
+func (c *ChunkCacheInMemory) GetChunk(fileId string) []byte {
+	item := c.cache.Get(fileId)
+	if item == nil {
+		return nil
+	}
+	data := item.Value().([]byte)
+	item.Extend(time.Hour)
+	return data
+}
+
+func (c *ChunkCacheInMemory) SetChunk(fileId string, data []byte) {
+	c.cache.Set(fileId, data, time.Hour)
+}
diff --git a/weed/util/chunk_cache/chunk_cache_on_disk.go b/weed/util/chunk_cache/chunk_cache_on_disk.go
new file mode 100644
index 000000000..2c7ef8d39
--- /dev/null
+++ b/weed/util/chunk_cache/chunk_cache_on_disk.go
@@ -0,0 +1,145 @@
+package chunk_cache
+
+import (
+	"fmt"
+	"os"
+	"time"
+
+	"github.com/syndtr/goleveldb/leveldb/opt"
+
+	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/storage"
+	"github.com/chrislusf/seaweedfs/weed/storage/backend"
+	"github.com/chrislusf/seaweedfs/weed/storage/types"
+	"github.com/chrislusf/seaweedfs/weed/util"
+)
+
+// This implements an on disk cache
+// The entries are an FIFO with a size limit
+
+type ChunkCacheVolume struct {
+	DataBackend backend.BackendStorageFile
+	nm          storage.NeedleMapper
+	fileName    string
+	smallBuffer []byte
+	sizeLimit   int64
+	lastModTime time.Time
+	fileSize    int64
+}
+
+func LoadOrCreateChunkCacheVolume(fileName string, preallocate int64) (*ChunkCacheVolume, error) {
+
+	v := &ChunkCacheVolume{
+		smallBuffer: make([]byte, types.NeedlePaddingSize),
+		fileName:    fileName,
+		sizeLimit:   preallocate,
+	}
+
+	var err error
+
+	if exists, canRead, canWrite, modTime, fileSize := util.CheckFile(v.fileName + ".dat"); exists {
+		if !canRead {
+			return nil, fmt.Errorf("cannot read cache file %s.dat", v.fileName)
+		}
+		if !canWrite {
+			return nil, fmt.Errorf("cannot write cache file %s.dat", v.fileName)
+		}
+		if dataFile, err := os.OpenFile(v.fileName+".dat", os.O_RDWR|os.O_CREATE, 0644); err != nil {
+			return nil, fmt.Errorf("cannot create cache file %s.dat: %v", v.fileName, err)
+		} else {
+			v.DataBackend = backend.NewDiskFile(dataFile)
+			v.lastModTime = modTime
+			v.fileSize = fileSize
+		}
+	} else {
+		if v.DataBackend, err = backend.CreateVolumeFile(v.fileName+".dat", preallocate, 0); err != nil {
+			return nil, fmt.Errorf("cannot create cache file %s.dat: %v", v.fileName, err)
+		}
+		v.lastModTime = time.Now()
+	}
+
+	var indexFile *os.File
+	if indexFile, err = os.OpenFile(v.fileName+".idx", os.O_RDWR|os.O_CREATE, 0644); err != nil {
+		return nil, fmt.Errorf("cannot write cache index %s.idx: %v", v.fileName, err)
+	}
+
+	glog.V(0).Infoln("loading leveldb", v.fileName+".ldb")
+	opts := &opt.Options{
+		BlockCacheCapacity:            2 * 1024 * 1024, // default value is 8MiB
+		WriteBuffer:                   1 * 1024 * 1024, // default value is 4MiB
+		CompactionTableSizeMultiplier: 10,              // default value is 1
+	}
+	if v.nm, err = storage.NewLevelDbNeedleMap(v.fileName+".ldb", indexFile, opts); err != nil {
+		return nil, fmt.Errorf("loading leveldb %s error: %v", v.fileName+".ldb", err)
+	}
+
+	return v, nil
+
+}
+
+func (v *ChunkCacheVolume) Shutdown() {
+	if v.DataBackend != nil {
+		v.DataBackend.Close()
+		v.DataBackend = nil
+	}
+	if v.nm != nil {
+		v.nm.Close()
+		v.nm = nil
+	}
+}
+
+func (v *ChunkCacheVolume) destroy() {
+	v.Shutdown()
+	os.Remove(v.fileName + ".dat")
+	os.Remove(v.fileName + ".idx")
+	os.RemoveAll(v.fileName + ".ldb")
+}
+
+func (v *ChunkCacheVolume) Reset() (*ChunkCacheVolume, error) {
+	v.destroy()
+	return LoadOrCreateChunkCacheVolume(v.fileName, v.sizeLimit)
+}
+
+func (v *ChunkCacheVolume) GetNeedle(key types.NeedleId) ([]byte, error) {
+
+	nv, ok := v.nm.Get(key)
+	if !ok {
+		return nil, storage.ErrorNotFound
+	}
+	data := make([]byte, nv.Size)
+	if readSize, readErr := v.DataBackend.ReadAt(data, nv.Offset.ToAcutalOffset()); readErr != nil {
+		return nil, fmt.Errorf("read %s.dat [%d,%d): %v",
+			v.fileName, nv.Offset.ToAcutalOffset(), nv.Offset.ToAcutalOffset()+int64(nv.Size), readErr)
+	} else {
+		if readSize != int(nv.Size) {
+			return nil, fmt.Errorf("read %d, expected %d", readSize, nv.Size)
+		}
+	}
+
+	return data, nil
+}
+
+func (v *ChunkCacheVolume) WriteNeedle(key types.NeedleId, data []byte) error {
+
+	offset := v.fileSize
+
+	written, err := v.DataBackend.WriteAt(data, offset)
+	if err != nil {
+		return err
+	} else if written != len(data) {
+		return fmt.Errorf("partial written %d, expected %d", written, len(data))
+	}
+
+	v.fileSize += int64(written)
+	extraSize := written % types.NeedlePaddingSize
+	if extraSize != 0 {
+		v.DataBackend.WriteAt(v.smallBuffer[:types.NeedlePaddingSize-extraSize], offset+int64(written))
+		v.fileSize += int64(types.NeedlePaddingSize - extraSize)
+	}
+
+	if err := v.nm.Put(key, types.ToOffset(offset), uint32(len(data))); err != nil {
+		glog.V(4).Infof("failed to save in needle map %d: %v", key, err)
+	}
+
+	return nil
+}
diff --git a/weed/util/chunk_cache/chunk_cache_on_disk_test.go b/weed/util/chunk_cache/chunk_cache_on_disk_test.go
new file mode 100644
index 000000000..256b10139
--- /dev/null
+++ b/weed/util/chunk_cache/chunk_cache_on_disk_test.go
@@ -0,0 +1,58 @@
+package chunk_cache
+
+import (
+	"bytes"
+	"fmt"
+	"io/ioutil"
+	"math/rand"
+	"os"
+	"testing"
+)
+
+func TestOnDisk(t *testing.T) {
+
+	tmpDir, _ := ioutil.TempDir("", "c")
+	defer os.RemoveAll(tmpDir)
+
+	totalDiskSizeMb := int64(6)
+	segmentCount := 2
+
+	cache := NewChunkCache(0, tmpDir, totalDiskSizeMb, segmentCount)
+
+	writeCount := 5
+	type test_data struct {
+		data []byte
+		fileId string
+	}
+	testData := make([]*test_data, writeCount)
+	for i:=0;i<writeCount;i++{
+		buff := make([]byte, 1024*1024)
+		rand.Read(buff)
+		testData[i] = &test_data{
+			data:   buff,
+			fileId: fmt.Sprintf("1,%daabbccdd", i+1),
+		}
+		cache.SetChunk(testData[i].fileId, testData[i].data)
+	}
+
+	for i:=0;i<writeCount;i++{
+		data := cache.GetChunk(testData[i].fileId)
+		if bytes.Compare(data, testData[i].data) != 0 {
+			t.Errorf("failed to write to and read from cache: %d", i)
+		}
+	}
+
+	cache.Shutdown()
+
+	cache = NewChunkCache(0, tmpDir, totalDiskSizeMb, segmentCount)
+
+	for i:=0;i<writeCount;i++{
+		data := cache.GetChunk(testData[i].fileId)
+		if bytes.Compare(data, testData[i].data) != 0 {
+			t.Errorf("failed to write to and read from cache: %d", i)
+		}
+	}
+
+	cache.Shutdown()
+
+}

From 5987810e5eb56611389451029fb85ff1fd700996 Mon Sep 17 00:00:00 2001
From: Chris Lu <chris.lu@gmail.com>
Date: Sat, 11 Apr 2020 21:22:52 -0700
Subject: [PATCH 40/81] set default chunk size to 16

---
 weed/command/mount.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/weed/command/mount.go b/weed/command/mount.go
index 6165402b4..661ddfe06 100644
--- a/weed/command/mount.go
+++ b/weed/command/mount.go
@@ -37,7 +37,7 @@ func init() {
 	mountOptions.collection = cmdMount.Flag.String("collection", "", "collection to create the files")
 	mountOptions.replication = cmdMount.Flag.String("replication", "", "replication(e.g. 000, 001) to create to files. If empty, let filer decide.")
 	mountOptions.ttlSec = cmdMount.Flag.Int("ttl", 0, "file ttl in seconds")
-	mountOptions.chunkSizeLimitMB = cmdMount.Flag.Int("chunkSizeLimitMB", 4, "local write buffer size, also chunk large files")
+	mountOptions.chunkSizeLimitMB = cmdMount.Flag.Int("chunkSizeLimitMB", 16, "local write buffer size, also chunk large files")
 	mountOptions.cacheDir = cmdMount.Flag.String("cacheDir", os.TempDir(), "local cache directory for file chunks")
 	mountOptions.cacheSizeMB = cmdMount.Flag.Int64("cacheCapacityMB", 1000, "local cache capacity in MB")
 	mountOptions.dataCenter = cmdMount.Flag.String("dataCenter", "", "prefer to write to the data center")

From e4af63a721f62c280596f626302b64729baeaee9 Mon Sep 17 00:00:00 2001
From: Chris Lu <chris.lu@gmail.com>
Date: Sat, 11 Apr 2020 21:39:16 -0700
Subject: [PATCH 41/81] volume server: accept fsync=true in write requests

---
 weed/server/volume_grpc_tail.go    |  2 +-
 weed/storage/store.go              |  4 ++--
 weed/storage/volume_read_write.go  |  7 ++++++-
 weed/storage/volume_vacuum_test.go |  2 +-
 weed/topology/store_replicate.go   | 10 +++++++++-
 5 files changed, 19 insertions(+), 6 deletions(-)

diff --git a/weed/server/volume_grpc_tail.go b/weed/server/volume_grpc_tail.go
index c3b66c5e7..2dde5b69c 100644
--- a/weed/server/volume_grpc_tail.go
+++ b/weed/server/volume_grpc_tail.go
@@ -90,7 +90,7 @@ func (vs *VolumeServer) VolumeTailReceiver(ctx context.Context, req *volume_serv
 	defer glog.V(1).Infof("receive tailing volume %d finished", v.Id)
 
 	return resp, operation.TailVolumeFromSource(req.SourceVolumeServer, vs.grpcDialOption, v.Id, req.SinceNs, int(req.IdleTimeoutSeconds), func(n *needle.Needle) error {
-		_, err := vs.store.WriteVolumeNeedle(v.Id, n)
+		_, err := vs.store.WriteVolumeNeedle(v.Id, n, false)
 		return err
 	})
 
diff --git a/weed/storage/store.go b/weed/storage/store.go
index d6b623e63..64e437add 100644
--- a/weed/storage/store.go
+++ b/weed/storage/store.go
@@ -252,7 +252,7 @@ func (s *Store) Close() {
 	}
 }
 
-func (s *Store) WriteVolumeNeedle(i needle.VolumeId, n *needle.Needle) (isUnchanged bool, err error) {
+func (s *Store) WriteVolumeNeedle(i needle.VolumeId, n *needle.Needle, fsync bool) (isUnchanged bool, err error) {
 	if v := s.findVolume(i); v != nil {
 		if v.IsReadOnly() {
 			err = fmt.Errorf("volume %d is read only", i)
@@ -260,7 +260,7 @@ func (s *Store) WriteVolumeNeedle(i needle.VolumeId, n *needle.Needle) (isUnchan
 		}
 		// using len(n.Data) here instead of n.Size before n.Size is populated in v.writeNeedle(n)
 		if MaxPossibleVolumeSize >= v.ContentSize()+uint64(needle.GetActualSize(uint32(len(n.Data)), v.Version())) {
-			_, _, isUnchanged, err = v.writeNeedle(n)
+			_, _, isUnchanged, err = v.writeNeedle(n, fsync)
 		} else {
 			err = fmt.Errorf("volume size limit %d exceeded! current size is %d", s.GetVolumeSizeLimit(), v.ContentSize())
 		}
diff --git a/weed/storage/volume_read_write.go b/weed/storage/volume_read_write.go
index ac6154cef..bb0421724 100644
--- a/weed/storage/volume_read_write.go
+++ b/weed/storage/volume_read_write.go
@@ -63,7 +63,7 @@ func (v *Volume) Destroy() (err error) {
 	return
 }
 
-func (v *Volume) writeNeedle(n *needle.Needle) (offset uint64, size uint32, isUnchanged bool, err error) {
+func (v *Volume) writeNeedle(n *needle.Needle, fsync bool) (offset uint64, size uint32, isUnchanged bool, err error) {
 	// glog.V(4).Infof("writing needle %s", needle.NewFileIdFromNeedle(v.Id, n).String())
 	v.dataFileAccessLock.Lock()
 	defer v.dataFileAccessLock.Unlock()
@@ -98,6 +98,11 @@ func (v *Volume) writeNeedle(n *needle.Needle) (offset uint64, size uint32, isUn
 	if offset, size, _, err = n.Append(v.DataBackend, v.Version()); err != nil {
 		return
 	}
+	if fsync {
+		if err = v.DataBackend.Sync(); err != nil {
+			return
+		}
+	}
 	v.lastAppendAtNs = n.AppendAtNs
 
 	// add to needle map
diff --git a/weed/storage/volume_vacuum_test.go b/weed/storage/volume_vacuum_test.go
index 51f04c8b1..5d0fcbe31 100644
--- a/weed/storage/volume_vacuum_test.go
+++ b/weed/storage/volume_vacuum_test.go
@@ -129,7 +129,7 @@ func TestCompaction(t *testing.T) {
 }
 func doSomeWritesDeletes(i int, v *Volume, t *testing.T, infos []*needleInfo) {
 	n := newRandomNeedle(uint64(i))
-	_, size, _, err := v.writeNeedle(n)
+	_, size, _, err := v.writeNeedle(n, false)
 	if err != nil {
 		t.Fatalf("write file %d: %v", i, err)
 	}
diff --git a/weed/topology/store_replicate.go b/weed/topology/store_replicate.go
index 6f043a601..236f8d773 100644
--- a/weed/topology/store_replicate.go
+++ b/weed/topology/store_replicate.go
@@ -22,8 +22,10 @@ func ReplicatedWrite(masterNode string, s *storage.Store, volumeId needle.Volume
 	//check JWT
 	jwt := security.GetJwt(r)
 
+	// check whether this is a replicated write request
 	var remoteLocations []operation.Location
 	if r.FormValue("type") != "replicate" {
+		// this is the initial request
 		remoteLocations, err = getWritableRemoteReplications(s, volumeId, masterNode)
 		if err != nil {
 			glog.V(0).Infoln(err)
@@ -31,8 +33,14 @@ func ReplicatedWrite(masterNode string, s *storage.Store, volumeId needle.Volume
 		}
 	}
 
+	// read fsync value
+	fsync := false
+	if r.FormValue("fsync") == "true" {
+		fsync = true
+	}
+
 	if s.GetVolume(volumeId) != nil {
-		isUnchanged, err = s.WriteVolumeNeedle(volumeId, n)
+		isUnchanged, err = s.WriteVolumeNeedle(volumeId, n, fsync)
 		if err != nil {
 			err = fmt.Errorf("failed to write to local disk: %v", err)
 			glog.V(0).Infoln(err)

From 1c65656fb4a64ae739b9a23a2f97f4032182015c Mon Sep 17 00:00:00 2001
From: Chris Lu <chris.lu@gmail.com>
Date: Sat, 11 Apr 2020 23:37:10 -0700
Subject: [PATCH 42/81] s3: add option to fsync buckets

---
 weed/command/scaffold.go                      |  4 ++++
 weed/filer2/filer.go                          |  1 +
 weed/filer2/filer_buckets.go                  | 19 +++++++++++++------
 weed/server/filer_grpc_server.go              |  2 +-
 weed/server/filer_server.go                   |  3 ++-
 weed/server/filer_server_handlers_write.go    | 17 ++++++++++-------
 .../filer_server_handlers_write_autochunk.go  |  8 ++++----
 .../filer_server_handlers_write_cipher.go     |  4 ++--
 8 files changed, 37 insertions(+), 21 deletions(-)

diff --git a/weed/command/scaffold.go b/weed/command/scaffold.go
index 468476ae4..109f0c04f 100644
--- a/weed/command/scaffold.go
+++ b/weed/command/scaffold.go
@@ -76,6 +76,10 @@ const (
 recursive_delete = false
 # directories under this folder will be automatically creating a separate bucket
 buckets_folder = "/buckets"
+buckets_fsync = [          # a list of buckets with all write requests fsync=true
+	"important_bucket",
+	"should_always_fsync",
+]
 # directories under this folder will be store message queue data
 queues_folder = "/queues"
 
diff --git a/weed/filer2/filer.go b/weed/filer2/filer.go
index dde08a092..8ad4fdc81 100644
--- a/weed/filer2/filer.go
+++ b/weed/filer2/filer.go
@@ -33,6 +33,7 @@ type Filer struct {
 	GrpcDialOption      grpc.DialOption
 	DirBucketsPath      string
 	DirQueuesPath       string
+	FsyncBuckets        []string
 	buckets             *FilerBuckets
 	Cipher              bool
 	metaLogBuffer       *log_buffer.LogBuffer
diff --git a/weed/filer2/filer_buckets.go b/weed/filer2/filer_buckets.go
index 3fc4afdab..7a57e7ee1 100644
--- a/weed/filer2/filer_buckets.go
+++ b/weed/filer2/filer_buckets.go
@@ -13,6 +13,7 @@ type BucketName string
 type BucketOption struct {
 	Name        BucketName
 	Replication string
+	fsync       bool
 }
 type FilerBuckets struct {
 	dirBucketsPath string
@@ -20,36 +21,42 @@ type FilerBuckets struct {
 	sync.RWMutex
 }
 
-func (f *Filer) LoadBuckets(dirBucketsPath string) {
+func (f *Filer) LoadBuckets() {
 
 	f.buckets = &FilerBuckets{
 		buckets: make(map[BucketName]*BucketOption),
 	}
-	f.DirBucketsPath = dirBucketsPath
 
 	limit := math.MaxInt32
 
-	entries, err := f.ListDirectoryEntries(context.Background(), util.FullPath(dirBucketsPath), "", false, limit)
+	entries, err := f.ListDirectoryEntries(context.Background(), util.FullPath(f.DirBucketsPath), "", false, limit)
 
 	if err != nil {
 		glog.V(1).Infof("no buckets found: %v", err)
 		return
 	}
 
+	shouldFsyncMap := make(map[string]bool)
+	for _, bucket := range f.FsyncBuckets {
+		shouldFsyncMap[bucket] = true
+	}
+
 	glog.V(1).Infof("buckets found: %d", len(entries))
 
 	f.buckets.Lock()
 	for _, entry := range entries {
+		_, shouldFsnyc := shouldFsyncMap[entry.Name()]
 		f.buckets.buckets[BucketName(entry.Name())] = &BucketOption{
 			Name:        BucketName(entry.Name()),
 			Replication: entry.Replication,
+			fsync:       shouldFsnyc,
 		}
 	}
 	f.buckets.Unlock()
 
 }
 
-func (f *Filer) ReadBucketOption(buketName string) (replication string) {
+func (f *Filer) ReadBucketOption(buketName string) (replication string, fsync bool) {
 
 	f.buckets.RLock()
 	defer f.buckets.RUnlock()
@@ -57,9 +64,9 @@ func (f *Filer) ReadBucketOption(buketName string) (replication string) {
 	option, found := f.buckets.buckets[BucketName(buketName)]
 
 	if !found {
-		return ""
+		return "", false
 	}
-	return option.Replication
+	return option.Replication, option.fsync
 
 }
 
diff --git a/weed/server/filer_grpc_server.go b/weed/server/filer_grpc_server.go
index 06889fbcb..ead246eb8 100644
--- a/weed/server/filer_grpc_server.go
+++ b/weed/server/filer_grpc_server.go
@@ -232,7 +232,7 @@ func (fs *FilerServer) AssignVolume(ctx context.Context, req *filer_pb.AssignVol
 	if req.TtlSec > 0 {
 		ttlStr = strconv.Itoa(int(req.TtlSec))
 	}
-	collection, replication := fs.detectCollection(req.ParentPath, req.Collection, req.Replication)
+	collection, replication, _ := fs.detectCollection(req.ParentPath, req.Collection, req.Replication)
 
 	var altRequest *operation.VolumeAssignRequest
 
diff --git a/weed/server/filer_server.go b/weed/server/filer_server.go
index 52dfbaf88..cf8669202 100644
--- a/weed/server/filer_server.go
+++ b/weed/server/filer_server.go
@@ -95,6 +95,7 @@ func NewFilerServer(defaultMux, readonlyMux *http.ServeMux, option *FilerOption)
 	v.SetDefault("filer.options.queues_folder", "/queues")
 	fs.filer.DirBucketsPath = v.GetString("filer.options.buckets_folder")
 	fs.filer.DirQueuesPath = v.GetString("filer.options.queues_folder")
+	fs.filer.FsyncBuckets = v.GetStringSlice("filer.options.buckets_fsync")
 	fs.filer.LoadConfiguration(v)
 
 	notification.LoadConfiguration(v, "notification.")
@@ -107,7 +108,7 @@ func NewFilerServer(defaultMux, readonlyMux *http.ServeMux, option *FilerOption)
 		readonlyMux.HandleFunc("/", fs.readonlyFilerHandler)
 	}
 
-	fs.filer.LoadBuckets(fs.filer.DirBucketsPath)
+	fs.filer.LoadBuckets()
 
 	util.OnInterrupt(func() {
 		fs.filer.Shutdown()
diff --git a/weed/server/filer_server_handlers_write.go b/weed/server/filer_server_handlers_write.go
index ce184875c..74a558e22 100644
--- a/weed/server/filer_server_handlers_write.go
+++ b/weed/server/filer_server_handlers_write.go
@@ -40,7 +40,7 @@ type FilerPostResult struct {
 	Url   string `json:"url,omitempty"`
 }
 
-func (fs *FilerServer) assignNewFileInfo(w http.ResponseWriter, r *http.Request, replication, collection, dataCenter, ttlString string) (fileId, urlLocation string, auth security.EncodedJwt, err error) {
+func (fs *FilerServer) assignNewFileInfo(w http.ResponseWriter, r *http.Request, replication, collection, dataCenter, ttlString string, fsync bool) (fileId, urlLocation string, auth security.EncodedJwt, err error) {
 
 	stats.FilerRequestCounter.WithLabelValues("assign").Inc()
 	start := time.Now()
@@ -73,6 +73,9 @@ func (fs *FilerServer) assignNewFileInfo(w http.ResponseWriter, r *http.Request,
 	}
 	fileId = assignResult.Fid
 	urlLocation = "http://" + assignResult.Url + "/" + assignResult.Fid
+	if fsync {
+		urlLocation += "?fsync=true"
+	}
 	auth = assignResult.Auth
 	return
 }
@@ -82,7 +85,7 @@ func (fs *FilerServer) PostHandler(w http.ResponseWriter, r *http.Request) {
 	ctx := context.Background()
 
 	query := r.URL.Query()
-	collection, replication := fs.detectCollection(r.RequestURI, query.Get("collection"), query.Get("replication"))
+	collection, replication, fsync := fs.detectCollection(r.RequestURI, query.Get("collection"), query.Get("replication"))
 	dataCenter := query.Get("dataCenter")
 	if dataCenter == "" {
 		dataCenter = fs.option.DataCenter
@@ -96,12 +99,12 @@ func (fs *FilerServer) PostHandler(w http.ResponseWriter, r *http.Request) {
 		ttlSeconds = int32(ttl.Minutes()) * 60
 	}
 
-	if autoChunked := fs.autoChunk(ctx, w, r, replication, collection, dataCenter, ttlSeconds, ttlString); autoChunked {
+	if autoChunked := fs.autoChunk(ctx, w, r, replication, collection, dataCenter, ttlSeconds, ttlString, fsync); autoChunked {
 		return
 	}
 
 	if fs.option.Cipher {
-		reply, err := fs.encrypt(ctx, w, r, replication, collection, dataCenter, ttlSeconds, ttlString)
+		reply, err := fs.encrypt(ctx, w, r, replication, collection, dataCenter, ttlSeconds, ttlString, fsync)
 		if err != nil {
 			writeJsonError(w, r, http.StatusInternalServerError, err)
 		} else if reply != nil {
@@ -111,7 +114,7 @@ func (fs *FilerServer) PostHandler(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	fileId, urlLocation, auth, err := fs.assignNewFileInfo(w, r, replication, collection, dataCenter, ttlString)
+	fileId, urlLocation, auth, err := fs.assignNewFileInfo(w, r, replication, collection, dataCenter, ttlString, fsync)
 
 	if err != nil || fileId == "" || urlLocation == "" {
 		glog.V(0).Infof("fail to allocate volume for %s, collection:%s, datacenter:%s", r.URL.Path, collection, dataCenter)
@@ -327,7 +330,7 @@ func (fs *FilerServer) DeleteHandler(w http.ResponseWriter, r *http.Request) {
 	w.WriteHeader(http.StatusNoContent)
 }
 
-func (fs *FilerServer) detectCollection(requestURI, qCollection, qReplication string) (collection, replication string) {
+func (fs *FilerServer) detectCollection(requestURI, qCollection, qReplication string) (collection, replication string, fsync bool) {
 	// default
 	collection = fs.option.Collection
 	replication = fs.option.DefaultReplication
@@ -350,7 +353,7 @@ func (fs *FilerServer) detectCollection(requestURI, qCollection, qReplication st
 		if t > 0 {
 			collection = bucketAndObjectKey[:t]
 		}
-		replication = fs.filer.ReadBucketOption(collection)
+		replication, fsync = fs.filer.ReadBucketOption(collection)
 	}
 
 	return
diff --git a/weed/server/filer_server_handlers_write_autochunk.go b/weed/server/filer_server_handlers_write_autochunk.go
index d86530bb8..4c371a9a5 100644
--- a/weed/server/filer_server_handlers_write_autochunk.go
+++ b/weed/server/filer_server_handlers_write_autochunk.go
@@ -21,7 +21,7 @@ import (
 )
 
 func (fs *FilerServer) autoChunk(ctx context.Context, w http.ResponseWriter, r *http.Request,
-	replication string, collection string, dataCenter string, ttlSec int32, ttlString string) bool {
+	replication string, collection string, dataCenter string, ttlSec int32, ttlString string, fsync bool) bool {
 	if r.Method != "POST" {
 		glog.V(4).Infoln("AutoChunking not supported for method", r.Method)
 		return false
@@ -57,7 +57,7 @@ func (fs *FilerServer) autoChunk(ctx context.Context, w http.ResponseWriter, r *
 		return false
 	}
 
-	reply, err := fs.doAutoChunk(ctx, w, r, contentLength, chunkSize, replication, collection, dataCenter, ttlSec, ttlString)
+	reply, err := fs.doAutoChunk(ctx, w, r, contentLength, chunkSize, replication, collection, dataCenter, ttlSec, ttlString, fsync)
 	if err != nil {
 		writeJsonError(w, r, http.StatusInternalServerError, err)
 	} else if reply != nil {
@@ -67,7 +67,7 @@ func (fs *FilerServer) autoChunk(ctx context.Context, w http.ResponseWriter, r *
 }
 
 func (fs *FilerServer) doAutoChunk(ctx context.Context, w http.ResponseWriter, r *http.Request,
-	contentLength int64, chunkSize int32, replication string, collection string, dataCenter string, ttlSec int32, ttlString string) (filerResult *FilerPostResult, replyerr error) {
+	contentLength int64, chunkSize int32, replication string, collection string, dataCenter string, ttlSec int32, ttlString string, fsync bool) (filerResult *FilerPostResult, replyerr error) {
 
 	stats.FilerRequestCounter.WithLabelValues("postAutoChunk").Inc()
 	start := time.Now()
@@ -102,7 +102,7 @@ func (fs *FilerServer) doAutoChunk(ctx context.Context, w http.ResponseWriter, r
 		limitedReader := io.LimitReader(partReader, int64(chunkSize))
 
 		// assign one file id for one chunk
-		fileId, urlLocation, auth, assignErr := fs.assignNewFileInfo(w, r, replication, collection, dataCenter, ttlString)
+		fileId, urlLocation, auth, assignErr := fs.assignNewFileInfo(w, r, replication, collection, dataCenter, ttlString, fsync)
 		if assignErr != nil {
 			return nil, assignErr
 		}
diff --git a/weed/server/filer_server_handlers_write_cipher.go b/weed/server/filer_server_handlers_write_cipher.go
index cbcf8a05c..ebf277984 100644
--- a/weed/server/filer_server_handlers_write_cipher.go
+++ b/weed/server/filer_server_handlers_write_cipher.go
@@ -17,9 +17,9 @@ import (
 
 // handling single chunk POST or PUT upload
 func (fs *FilerServer) encrypt(ctx context.Context, w http.ResponseWriter, r *http.Request,
-	replication string, collection string, dataCenter string, ttlSeconds int32, ttlString string) (filerResult *FilerPostResult, err error) {
+	replication string, collection string, dataCenter string, ttlSeconds int32, ttlString string, fsync bool) (filerResult *FilerPostResult, err error) {
 
-	fileId, urlLocation, auth, err := fs.assignNewFileInfo(w, r, replication, collection, dataCenter, ttlString)
+	fileId, urlLocation, auth, err := fs.assignNewFileInfo(w, r, replication, collection, dataCenter, ttlString, fsync)
 
 	if err != nil || fileId == "" || urlLocation == "" {
 		return nil, fmt.Errorf("fail to allocate volume for %s, collection:%s, datacenter:%s", r.URL.Path, collection, dataCenter)

From 211d87cf4c6afed1704733654ad3944bf40dd2bd Mon Sep 17 00:00:00 2001
From: Chris Lu <chris.lu@gmail.com>
Date: Sun, 12 Apr 2020 00:52:54 -0700
Subject: [PATCH 43/81] mount: option to disable caching

---
 weed/command/mount.go |  2 +-
 weed/filesys/dir.go   |  4 ++--
 weed/filesys/wfs.go   | 11 ++++++-----
 3 files changed, 9 insertions(+), 8 deletions(-)

diff --git a/weed/command/mount.go b/weed/command/mount.go
index 661ddfe06..efa4650ab 100644
--- a/weed/command/mount.go
+++ b/weed/command/mount.go
@@ -39,7 +39,7 @@ func init() {
 	mountOptions.ttlSec = cmdMount.Flag.Int("ttl", 0, "file ttl in seconds")
 	mountOptions.chunkSizeLimitMB = cmdMount.Flag.Int("chunkSizeLimitMB", 16, "local write buffer size, also chunk large files")
 	mountOptions.cacheDir = cmdMount.Flag.String("cacheDir", os.TempDir(), "local cache directory for file chunks")
-	mountOptions.cacheSizeMB = cmdMount.Flag.Int64("cacheCapacityMB", 1000, "local cache capacity in MB")
+	mountOptions.cacheSizeMB = cmdMount.Flag.Int64("cacheCapacityMB", 1000, "local cache capacity in MB (0 will disable cache)")
 	mountOptions.dataCenter = cmdMount.Flag.String("dataCenter", "", "prefer to write to the data center")
 	mountOptions.allowOthers = cmdMount.Flag.Bool("allowOthers", true, "allows other users to access the file system")
 	mountOptions.umaskString = cmdMount.Flag.String("umask", "022", "octal umask, e.g., 022, 0111")
diff --git a/weed/filesys/dir.go b/weed/filesys/dir.go
index c892b4f91..46e8aebb4 100644
--- a/weed/filesys/dir.go
+++ b/weed/filesys/dir.go
@@ -58,7 +58,7 @@ func (dir *Dir) Attr(ctx context.Context, attr *fuse.Attr) error {
 	attr.Gid = dir.entry.Attributes.Gid
 	attr.Uid = dir.entry.Attributes.Uid
 
-	glog.V(3).Infof("dir Attr %s, attr: %+v", dir.FullPath(), attr)
+	glog.V(4).Infof("dir Attr %s, attr: %+v", dir.FullPath(), attr)
 
 	return nil
 }
@@ -200,7 +200,7 @@ func (dir *Dir) Mkdir(ctx context.Context, req *fuse.MkdirRequest) (fs.Node, err
 
 func (dir *Dir) Lookup(ctx context.Context, req *fuse.LookupRequest, resp *fuse.LookupResponse) (node fs.Node, err error) {
 
-	glog.V(4).Infof("dir Lookup %s: %s", dir.FullPath(), req.Name)
+	glog.V(4).Infof("dir Lookup %s: %s by %s", dir.FullPath(), req.Name, req.Header.String())
 
 	fullFilePath := util.NewFullPath(dir.FullPath(), req.Name)
 	entry := dir.wfs.cacheGet(fullFilePath)
diff --git a/weed/filesys/wfs.go b/weed/filesys/wfs.go
index b2f68c030..64c5d5aa6 100644
--- a/weed/filesys/wfs.go
+++ b/weed/filesys/wfs.go
@@ -73,10 +73,6 @@ type statsCache struct {
 }
 
 func NewSeaweedFileSystem(option *Option) *WFS {
-	chunkCache := chunk_cache.NewChunkCache(256, option.CacheDir, option.CacheSizeMB, 4)
-	util.OnInterrupt(func() {
-		chunkCache.Shutdown()
-	})
 	wfs := &WFS{
 		option:                    option,
 		listDirectoryEntriesCache: ccache.New(ccache.Configure().MaxSize(option.DirListCacheLimit * 3).ItemsToPrune(100)),
@@ -86,7 +82,12 @@ func NewSeaweedFileSystem(option *Option) *WFS {
 				return make([]byte, option.ChunkSizeLimit)
 			},
 		},
-		chunkCache: chunkCache,
+	}
+	if option.CacheSizeMB > 0 {
+		wfs.chunkCache = chunk_cache.NewChunkCache(256, option.CacheDir, option.CacheSizeMB, 4)
+		util.OnInterrupt(func() {
+			wfs.chunkCache.Shutdown()
+		})
 	}
 
 	wfs.root = &Dir{name: wfs.option.FilerMountRootPath, wfs: wfs}

From b9b7da905ef9ef51d3e060ab1612becd63ab272d Mon Sep 17 00:00:00 2001
From: Chris Lu <chris.lu@gmail.com>
Date: Sun, 12 Apr 2020 01:00:12 -0700
Subject: [PATCH 44/81] handle nil chunk cache

---
 weed/util/chunk_cache/chunk_cache.go | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/weed/util/chunk_cache/chunk_cache.go b/weed/util/chunk_cache/chunk_cache.go
index 682f5185a..ead7a8d0b 100644
--- a/weed/util/chunk_cache/chunk_cache.go
+++ b/weed/util/chunk_cache/chunk_cache.go
@@ -47,6 +47,10 @@ func NewChunkCache(maxEntries int64, dir string, diskSizeMB int64, segmentCount
 }
 
 func (c *ChunkCache) GetChunk(fileId string) (data []byte) {
+	if c == nil {
+		return
+	}
+
 	c.RLock()
 	defer c.RUnlock()
 
@@ -76,6 +80,9 @@ func (c *ChunkCache) GetChunk(fileId string) (data []byte) {
 }
 
 func (c *ChunkCache) SetChunk(fileId string, data []byte) {
+	if c == nil {
+		return
+	}
 	c.Lock()
 	defer c.Unlock()
 
@@ -107,6 +114,9 @@ func (c *ChunkCache) SetChunk(fileId string, data []byte) {
 }
 
 func (c *ChunkCache) Shutdown() {
+	if c == nil {
+		return
+	}
 	c.Lock()
 	defer c.Unlock()
 	for _, diskCache := range c.diskCaches {

From 2a1f396df5abd47e7fc4a58c3bc39675e1e84e4f Mon Sep 17 00:00:00 2001
From: Chris Lu <chris.lu@gmail.com>
Date: Sun, 12 Apr 2020 01:06:50 -0700
Subject: [PATCH 45/81] avoid duplicated setting chunks into cache

---
 weed/util/chunk_cache/chunk_cache.go | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/weed/util/chunk_cache/chunk_cache.go b/weed/util/chunk_cache/chunk_cache.go
index ead7a8d0b..48e4bfb0d 100644
--- a/weed/util/chunk_cache/chunk_cache.go
+++ b/weed/util/chunk_cache/chunk_cache.go
@@ -54,6 +54,10 @@ func (c *ChunkCache) GetChunk(fileId string) (data []byte) {
 	c.RLock()
 	defer c.RUnlock()
 
+	return c.doGetChunk(fileId)
+}
+
+func (c *ChunkCache) doGetChunk(fileId string) (data []byte) {
 	if data = c.memCache.GetChunk(fileId); data != nil {
 		return data
 	}
@@ -86,6 +90,13 @@ func (c *ChunkCache) SetChunk(fileId string, data []byte) {
 	c.Lock()
 	defer c.Unlock()
 
+	if existingData := c.doGetChunk(fileId); len(existingData)==0{
+		c.doSetChunk(fileId, data)
+	}
+}
+
+func (c *ChunkCache) doSetChunk(fileId string, data []byte) {
+
 	c.memCache.SetChunk(fileId, data)
 
 	if len(c.diskCaches) == 0 {

From e613695af794e5509ea454920f78063f453f16f2 Mon Sep 17 00:00:00 2001
From: Chris Lu <chris.lu@gmail.com>
Date: Sun, 12 Apr 2020 01:13:57 -0700
Subject: [PATCH 46/81] a little optimization

---
 weed/filer2/reader_at.go | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/weed/filer2/reader_at.go b/weed/filer2/reader_at.go
index 2782cc204..c1ad1677f 100644
--- a/weed/filer2/reader_at.go
+++ b/weed/filer2/reader_at.go
@@ -105,9 +105,11 @@ func (c *ChunkReadAt) fetchChunkData(chunkView *ChunkView) (data []byte, err err
 
 	// fmt.Printf("fetching %s [%d,%d)\n", chunkView.FileId, chunkView.LogicOffset, chunkView.LogicOffset+int64(chunkView.Size))
 
+	hasDataInCache := false
 	chunkData := c.chunkCache.GetChunk(chunkView.FileId)
 	if chunkData != nil {
 		glog.V(3).Infof("cache hit %s [%d,%d)", chunkView.FileId, chunkView.LogicOffset, chunkView.LogicOffset+int64(chunkView.Size))
+		hasDataInCache = true
 	} else {
 		chunkData, err = c.doFetchFullChunkData(chunkView.FileId, chunkView.CipherKey, chunkView.IsGzipped)
 		if err != nil {
@@ -121,7 +123,9 @@ func (c *ChunkReadAt) fetchChunkData(chunkView *ChunkView) (data []byte, err err
 
 	data = chunkData[chunkView.Offset : chunkView.Offset+int64(chunkView.Size)]
 
-	c.chunkCache.SetChunk(chunkView.FileId, chunkData)
+	if !hasDataInCache {
+		c.chunkCache.SetChunk(chunkView.FileId, chunkData)
+	}
 
 	return data, nil
 }

From b5a713be6843d8899cfae1021ac54a18b9315dd2 Mon Sep 17 00:00:00 2001
From: Chris Lu <chris.lu@gmail.com>
Date: Sun, 12 Apr 2020 02:50:41 -0700
Subject: [PATCH 47/81] filer store: add redis2 redis_cluster2 as default redis
 store

---
 weed/command/scaffold.go                    |   4 +-
 weed/filer2/redis2/redis_cluster_store.go   |  42 +++++
 weed/filer2/redis2/redis_store.go           |  36 +++++
 weed/filer2/redis2/universal_redis_store.go | 165 ++++++++++++++++++++
 weed/server/filer_server.go                 |   1 +
 5 files changed, 246 insertions(+), 2 deletions(-)
 create mode 100644 weed/filer2/redis2/redis_cluster_store.go
 create mode 100644 weed/filer2/redis2/redis_store.go
 create mode 100644 weed/filer2/redis2/universal_redis_store.go

diff --git a/weed/command/scaffold.go b/weed/command/scaffold.go
index 109f0c04f..67fa8dcf3 100644
--- a/weed/command/scaffold.go
+++ b/weed/command/scaffold.go
@@ -143,13 +143,13 @@ hosts=[
 	"localhost:9042",
 ]
 
-[redis]
+[redis2]
 enabled = false
 address  = "localhost:6379"
 password = ""
 database = 0
 
-[redis_cluster]
+[redis_cluster2]
 enabled = false
 addresses = [
     "localhost:30001",
diff --git a/weed/filer2/redis2/redis_cluster_store.go b/weed/filer2/redis2/redis_cluster_store.go
new file mode 100644
index 000000000..b252eabab
--- /dev/null
+++ b/weed/filer2/redis2/redis_cluster_store.go
@@ -0,0 +1,42 @@
+package redis2
+
+import (
+	"github.com/chrislusf/seaweedfs/weed/filer2"
+	"github.com/chrislusf/seaweedfs/weed/util"
+	"github.com/go-redis/redis"
+)
+
+func init() {
+	filer2.Stores = append(filer2.Stores, &RedisCluster2Store{})
+}
+
+type RedisCluster2Store struct {
+	UniversalRedis2Store
+}
+
+func (store *RedisCluster2Store) GetName() string {
+	return "redis_cluster2"
+}
+
+func (store *RedisCluster2Store) Initialize(configuration util.Configuration, prefix string) (err error) {
+
+	configuration.SetDefault(prefix+"useReadOnly", true)
+	configuration.SetDefault(prefix+"routeByLatency", true)
+
+	return store.initialize(
+		configuration.GetStringSlice(prefix+"addresses"),
+		configuration.GetString(prefix+"password"),
+		configuration.GetBool(prefix+"useReadOnly"),
+		configuration.GetBool(prefix+"routeByLatency"),
+	)
+}
+
+func (store *RedisCluster2Store) initialize(addresses []string, password string, readOnly, routeByLatency bool) (err error) {
+	store.Client = redis.NewClusterClient(&redis.ClusterOptions{
+		Addrs:          addresses,
+		Password:       password,
+		ReadOnly:       readOnly,
+		RouteByLatency: routeByLatency,
+	})
+	return
+}
diff --git a/weed/filer2/redis2/redis_store.go b/weed/filer2/redis2/redis_store.go
new file mode 100644
index 000000000..1e2a20043
--- /dev/null
+++ b/weed/filer2/redis2/redis_store.go
@@ -0,0 +1,36 @@
+package redis2
+
+import (
+	"github.com/chrislusf/seaweedfs/weed/filer2"
+	"github.com/chrislusf/seaweedfs/weed/util"
+	"github.com/go-redis/redis"
+)
+
+func init() {
+	filer2.Stores = append(filer2.Stores, &Redis2Store{})
+}
+
+type Redis2Store struct {
+	UniversalRedis2Store
+}
+
+func (store *Redis2Store) GetName() string {
+	return "redis2"
+}
+
+func (store *Redis2Store) Initialize(configuration util.Configuration, prefix string) (err error) {
+	return store.initialize(
+		configuration.GetString(prefix+"address"),
+		configuration.GetString(prefix+"password"),
+		configuration.GetInt(prefix+"database"),
+	)
+}
+
+func (store *Redis2Store) initialize(hostPort string, password string, database int) (err error) {
+	store.Client = redis.NewClient(&redis.Options{
+		Addr:     hostPort,
+		Password: password,
+		DB:       database,
+	})
+	return
+}
diff --git a/weed/filer2/redis2/universal_redis_store.go b/weed/filer2/redis2/universal_redis_store.go
new file mode 100644
index 000000000..457ab50c6
--- /dev/null
+++ b/weed/filer2/redis2/universal_redis_store.go
@@ -0,0 +1,165 @@
+package redis2
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"github.com/go-redis/redis"
+
+	"github.com/chrislusf/seaweedfs/weed/filer2"
+	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
+	"github.com/chrislusf/seaweedfs/weed/util"
+)
+
+const (
+	DIR_LIST_MARKER = "\x00"
+)
+
+type UniversalRedis2Store struct {
+	Client redis.UniversalClient
+}
+
+func (store *UniversalRedis2Store) BeginTransaction(ctx context.Context) (context.Context, error) {
+	return ctx, nil
+}
+func (store *UniversalRedis2Store) CommitTransaction(ctx context.Context) error {
+	return nil
+}
+func (store *UniversalRedis2Store) RollbackTransaction(ctx context.Context) error {
+	return nil
+}
+
+func (store *UniversalRedis2Store) InsertEntry(ctx context.Context, entry *filer2.Entry) (err error) {
+
+	value, err := entry.EncodeAttributesAndChunks()
+	if err != nil {
+		return fmt.Errorf("encoding %s %+v: %v", entry.FullPath, entry.Attr, err)
+	}
+
+	_, err = store.Client.Set(string(entry.FullPath), value, time.Duration(entry.TtlSec)*time.Second).Result()
+
+	if err != nil {
+		return fmt.Errorf("persisting %s : %v", entry.FullPath, err)
+	}
+
+	dir, name := entry.FullPath.DirAndName()
+	if name != "" {
+		_, err = store.Client.ZAdd(genDirectoryListKey(dir), &redis.Z{Score: 0, Member: name}).Result()
+		if err != nil {
+			return fmt.Errorf("persisting %s in parent dir: %v", entry.FullPath, err)
+		}
+	}
+
+	return nil
+}
+
+func (store *UniversalRedis2Store) UpdateEntry(ctx context.Context, entry *filer2.Entry) (err error) {
+
+	return store.InsertEntry(ctx, entry)
+}
+
+func (store *UniversalRedis2Store) FindEntry(ctx context.Context, fullpath util.FullPath) (entry *filer2.Entry, err error) {
+
+	data, err := store.Client.Get(string(fullpath)).Result()
+	if err == redis.Nil {
+		return nil, filer_pb.ErrNotFound
+	}
+
+	if err != nil {
+		return nil, fmt.Errorf("get %s : %v", fullpath, err)
+	}
+
+	entry = &filer2.Entry{
+		FullPath: fullpath,
+	}
+	err = entry.DecodeAttributesAndChunks([]byte(data))
+	if err != nil {
+		return entry, fmt.Errorf("decode %s : %v", entry.FullPath, err)
+	}
+
+	return entry, nil
+}
+
+func (store *UniversalRedis2Store) DeleteEntry(ctx context.Context, fullpath util.FullPath) (err error) {
+
+	_, err = store.Client.Del(string(fullpath)).Result()
+
+	if err != nil {
+		return fmt.Errorf("delete %s : %v", fullpath, err)
+	}
+
+	dir, name := fullpath.DirAndName()
+	if name != "" {
+		_, err = store.Client.ZRem(genDirectoryListKey(dir), name).Result()
+		if err != nil {
+			return fmt.Errorf("delete %s in parent dir: %v", fullpath, err)
+		}
+	}
+
+	return nil
+}
+
+func (store *UniversalRedis2Store) DeleteFolderChildren(ctx context.Context, fullpath util.FullPath) (err error) {
+
+	members, err := store.Client.ZRange(genDirectoryListKey(string(fullpath)), 0, -1).Result()
+	if err != nil {
+		return fmt.Errorf("delete folder %s : %v", fullpath, err)
+	}
+
+	for _, fileName := range members {
+		path := util.NewFullPath(string(fullpath), fileName)
+		_, err = store.Client.Del(string(path)).Result()
+		if err != nil {
+			return fmt.Errorf("delete %s in parent dir: %v", fullpath, err)
+		}
+	}
+
+	return nil
+}
+
+func (store *UniversalRedis2Store) ListDirectoryEntries(ctx context.Context, fullpath util.FullPath, startFileName string, inclusive bool,
+	limit int) (entries []*filer2.Entry, err error) {
+
+	dirListKey := genDirectoryListKey(string(fullpath))
+	start := int64(0)
+	if startFileName != "" {
+		start, _ = store.Client.ZRank(dirListKey, startFileName).Result()
+		if !inclusive {
+			start++
+		}
+	}
+	members, err := store.Client.ZRange(dirListKey, start, start+int64(limit)-1).Result()
+	if err != nil {
+		return nil, fmt.Errorf("list %s : %v", fullpath, err)
+	}
+
+	// fetch entry meta
+	for _, fileName := range members {
+		path := util.NewFullPath(string(fullpath), fileName)
+		entry, err := store.FindEntry(ctx, path)
+		if err != nil {
+			glog.V(0).Infof("list %s : %v", path, err)
+		} else {
+			if entry.TtlSec > 0 {
+				if entry.Attr.Crtime.Add(time.Duration(entry.TtlSec) * time.Second).Before(time.Now()) {
+					store.Client.Del(string(path)).Result()
+					store.Client.ZRem(dirListKey, fileName).Result()
+					continue
+				}
+			}
+			entries = append(entries, entry)
+		}
+	}
+
+	return entries, err
+}
+
+func genDirectoryListKey(dir string) (dirList string) {
+	return dir + DIR_LIST_MARKER
+}
+
+func (store *UniversalRedis2Store) Shutdown() {
+	store.Client.Close()
+}
diff --git a/weed/server/filer_server.go b/weed/server/filer_server.go
index cf8669202..63bc8bd03 100644
--- a/weed/server/filer_server.go
+++ b/weed/server/filer_server.go
@@ -24,6 +24,7 @@ import (
 	_ "github.com/chrislusf/seaweedfs/weed/filer2/mysql"
 	_ "github.com/chrislusf/seaweedfs/weed/filer2/postgres"
 	_ "github.com/chrislusf/seaweedfs/weed/filer2/redis"
+	_ "github.com/chrislusf/seaweedfs/weed/filer2/redis2"
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/notification"
 	_ "github.com/chrislusf/seaweedfs/weed/notification/aws_sqs"

From 1c2e920fa2831d871ec55fed04818fddaaaacbd5 Mon Sep 17 00:00:00 2001
From: Chris Lu <chris.lu@gmail.com>
Date: Sun, 12 Apr 2020 03:11:19 -0700
Subject: [PATCH 48/81] fix compilation error

---
 go.mod                                      | 2 +-
 go.sum                                      | 2 ++
 weed/filer2/redis2/universal_redis_store.go | 7 ++-----
 3 files changed, 5 insertions(+), 6 deletions(-)

diff --git a/go.mod b/go.mod
index 15d7a3759..e408618f3 100644
--- a/go.mod
+++ b/go.mod
@@ -25,7 +25,7 @@ require (
 	github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4 // indirect
 	github.com/frankban/quicktest v1.7.2 // indirect
 	github.com/gabriel-vasile/mimetype v1.0.0
-	github.com/go-redis/redis v6.15.2+incompatible
+	github.com/go-redis/redis v6.15.7+incompatible
 	github.com/go-sql-driver/mysql v1.4.1
 	github.com/gocql/gocql v0.0.0-20190829130954-e163eff7a8c6
 	github.com/gogo/protobuf v1.2.2-0.20190730201129-28a6bbf47e48 // indirect
diff --git a/go.sum b/go.sum
index 5c1d52afd..fff96e5f6 100644
--- a/go.sum
+++ b/go.sum
@@ -131,6 +131,8 @@ github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9
 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
 github.com/go-redis/redis v6.15.2+incompatible h1:9SpNVG76gr6InJGxoZ6IuuxaCOQwDAhzyXg+Bs+0Sb4=
 github.com/go-redis/redis v6.15.2+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA=
+github.com/go-redis/redis v6.15.7+incompatible h1:3skhDh95XQMpnqeqNftPkQD9jL9e5e36z/1SUm6dy1U=
+github.com/go-redis/redis v6.15.7+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA=
 github.com/go-sql-driver/mysql v1.4.1 h1:g24URVg0OFbNUTx9qqY1IRZ9D9z3iPyi5zKhQZpNwpA=
 github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
diff --git a/weed/filer2/redis2/universal_redis_store.go b/weed/filer2/redis2/universal_redis_store.go
index 457ab50c6..420336b46 100644
--- a/weed/filer2/redis2/universal_redis_store.go
+++ b/weed/filer2/redis2/universal_redis_store.go
@@ -38,16 +38,13 @@ func (store *UniversalRedis2Store) InsertEntry(ctx context.Context, entry *filer
 		return fmt.Errorf("encoding %s %+v: %v", entry.FullPath, entry.Attr, err)
 	}
 
-	_, err = store.Client.Set(string(entry.FullPath), value, time.Duration(entry.TtlSec)*time.Second).Result()
-
-	if err != nil {
+	if err = store.Client.Set(string(entry.FullPath), value, time.Duration(entry.TtlSec)*time.Second).Err(); err != nil {
 		return fmt.Errorf("persisting %s : %v", entry.FullPath, err)
 	}
 
 	dir, name := entry.FullPath.DirAndName()
 	if name != "" {
-		_, err = store.Client.ZAdd(genDirectoryListKey(dir), &redis.Z{Score: 0, Member: name}).Result()
-		if err != nil {
+		if err = store.Client.ZAddNX(genDirectoryListKey(dir), redis.Z{Score: 0, Member: name}).Err(); err != nil {
 			return fmt.Errorf("persisting %s in parent dir: %v", entry.FullPath, err)
 		}
 	}

From 94e35cdb3552498b25824950bde94334c8b25331 Mon Sep 17 00:00:00 2001
From: Chris Lu <chris.lu@gmail.com>
Date: Sun, 12 Apr 2020 03:34:36 -0700
Subject: [PATCH 49/81] mount: fix

fix bug found by git bisect, but I do not understand why it can cause error!
---
 weed/util/chunk_cache/chunk_cache.go | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/weed/util/chunk_cache/chunk_cache.go b/weed/util/chunk_cache/chunk_cache.go
index 48e4bfb0d..9f2a0518f 100644
--- a/weed/util/chunk_cache/chunk_cache.go
+++ b/weed/util/chunk_cache/chunk_cache.go
@@ -90,9 +90,7 @@ func (c *ChunkCache) SetChunk(fileId string, data []byte) {
 	c.Lock()
 	defer c.Unlock()
 
-	if existingData := c.doGetChunk(fileId); len(existingData)==0{
-		c.doSetChunk(fileId, data)
-	}
+	c.doSetChunk(fileId, data)
 }
 
 func (c *ChunkCache) doSetChunk(fileId string, data []byte) {

From 6f948e48879ed0706b044bea0429d3fc48d6e8e1 Mon Sep 17 00:00:00 2001
From: Chris Lu <chris.lu@gmail.com>
Date: Sun, 12 Apr 2020 13:07:59 -0700
Subject: [PATCH 50/81] remove configurable topics folder location

---
 other/java/client/src/main/proto/filer.proto |   1 -
 weed/command/msg_broker.go                   |   6 +-
 weed/command/scaffold.go                     |   2 -
 weed/filer2/filer.go                         |   1 -
 weed/pb/filer.proto                          |   1 -
 weed/pb/filer_pb/filer.pb.go                 | 241 +++++++++----------
 weed/server/filer_grpc_server.go             |   1 -
 weed/server/filer_server.go                  |   2 -
 8 files changed, 117 insertions(+), 138 deletions(-)

diff --git a/other/java/client/src/main/proto/filer.proto b/other/java/client/src/main/proto/filer.proto
index 8f8176ff4..40316f58b 100644
--- a/other/java/client/src/main/proto/filer.proto
+++ b/other/java/client/src/main/proto/filer.proto
@@ -231,7 +231,6 @@ message GetFilerConfigurationResponse {
     string collection = 3;
     uint32 max_mb = 4;
     string dir_buckets = 5;
-    string dir_queues = 6;
     bool cipher = 7;
 }
 
diff --git a/weed/command/msg_broker.go b/weed/command/msg_broker.go
index 3e13b4730..093cff4a6 100644
--- a/weed/command/msg_broker.go
+++ b/weed/command/msg_broker.go
@@ -62,18 +62,14 @@ func (msgBrokerOpt *QueueOptions) startQueueServer() bool {
 		return false
 	}
 
-	filerQueuesPath := "/queues"
-
 	grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client")
 
 	for {
 		err = pb.WithGrpcFilerClient(filerGrpcAddress, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
-			resp, err := client.GetFilerConfiguration(context.Background(), &filer_pb.GetFilerConfigurationRequest{})
+			_, err := client.GetFilerConfiguration(context.Background(), &filer_pb.GetFilerConfigurationRequest{})
 			if err != nil {
 				return fmt.Errorf("get filer %s configuration: %v", filerGrpcAddress, err)
 			}
-			filerQueuesPath = resp.DirQueues
-			glog.V(0).Infof("Queue read filer queues dir: %s", filerQueuesPath)
 			return nil
 		})
 		if err != nil {
diff --git a/weed/command/scaffold.go b/weed/command/scaffold.go
index 67fa8dcf3..2c2bdc737 100644
--- a/weed/command/scaffold.go
+++ b/weed/command/scaffold.go
@@ -80,8 +80,6 @@ buckets_fsync = [          # a list of buckets with all write requests fsync=tru
 	"important_bucket",
 	"should_always_fsync",
 ]
-# directories under this folder will be store message queue data
-queues_folder = "/queues"
 
 ####################################################
 # The following are filer store options
diff --git a/weed/filer2/filer.go b/weed/filer2/filer.go
index 8ad4fdc81..df6379d26 100644
--- a/weed/filer2/filer.go
+++ b/weed/filer2/filer.go
@@ -32,7 +32,6 @@ type Filer struct {
 	fileIdDeletionQueue *util.UnboundedQueue
 	GrpcDialOption      grpc.DialOption
 	DirBucketsPath      string
-	DirQueuesPath       string
 	FsyncBuckets        []string
 	buckets             *FilerBuckets
 	Cipher              bool
diff --git a/weed/pb/filer.proto b/weed/pb/filer.proto
index 8f8176ff4..40316f58b 100644
--- a/weed/pb/filer.proto
+++ b/weed/pb/filer.proto
@@ -231,7 +231,6 @@ message GetFilerConfigurationResponse {
     string collection = 3;
     uint32 max_mb = 4;
     string dir_buckets = 5;
-    string dir_queues = 6;
     bool cipher = 7;
 }
 
diff --git a/weed/pb/filer_pb/filer.pb.go b/weed/pb/filer_pb/filer.pb.go
index 814856593..e428bb23a 100644
--- a/weed/pb/filer_pb/filer.pb.go
+++ b/weed/pb/filer_pb/filer.pb.go
@@ -1040,7 +1040,6 @@ type GetFilerConfigurationResponse struct {
 	Collection  string   `protobuf:"bytes,3,opt,name=collection" json:"collection,omitempty"`
 	MaxMb       uint32   `protobuf:"varint,4,opt,name=max_mb,json=maxMb" json:"max_mb,omitempty"`
 	DirBuckets  string   `protobuf:"bytes,5,opt,name=dir_buckets,json=dirBuckets" json:"dir_buckets,omitempty"`
-	DirQueues   string   `protobuf:"bytes,6,opt,name=dir_queues,json=dirQueues" json:"dir_queues,omitempty"`
 	Cipher      bool     `protobuf:"varint,7,opt,name=cipher" json:"cipher,omitempty"`
 }
 
@@ -1084,13 +1083,6 @@ func (m *GetFilerConfigurationResponse) GetDirBuckets() string {
 	return ""
 }
 
-func (m *GetFilerConfigurationResponse) GetDirQueues() string {
-	if m != nil {
-		return m.DirQueues
-	}
-	return ""
-}
-
 func (m *GetFilerConfigurationResponse) GetCipher() bool {
 	if m != nil {
 		return m.Cipher
@@ -1715,125 +1707,124 @@ var _SeaweedFiler_serviceDesc = grpc.ServiceDesc{
 func init() { proto.RegisterFile("filer.proto", fileDescriptor0) }
 
 var fileDescriptor0 = []byte{
-	// 1919 bytes of a gzipped FileDescriptorProto
+	// 1903 bytes of a gzipped FileDescriptorProto
 	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xb4, 0x58, 0x5f, 0x6f, 0xdc, 0xc6,
-	0x11, 0x37, 0xef, 0x74, 0x7f, 0x38, 0x77, 0x67, 0x4b, 0x7b, 0xb2, 0x73, 0x3e, 0x4b, 0xb6, 0x42,
+	0x11, 0x37, 0xef, 0x74, 0x7f, 0x38, 0x77, 0xe7, 0x48, 0x7b, 0x92, 0x73, 0x3e, 0x4b, 0xb6, 0x42,
 	0xd7, 0xa9, 0x0b, 0x1b, 0xaa, 0xa1, 0xa6, 0x40, 0xd2, 0xb4, 0x0f, 0xb6, 0x2c, 0xa5, 0x6e, 0x6c,
-	0xc5, 0xa5, 0xec, 0x22, 0x45, 0x81, 0x12, 0x14, 0xb9, 0xba, 0xdb, 0x8a, 0x47, 0x32, 0xbb, 0x4b,
-	0xfd, 0xc9, 0x53, 0xfb, 0x35, 0x0a, 0xf4, 0xa1, 0xdf, 0xa1, 0x8f, 0x45, 0x5f, 0x8a, 0x02, 0xfd,
-	0x1c, 0x7d, 0xec, 0xa7, 0x28, 0x76, 0x96, 0xe4, 0x2d, 0x8f, 0x27, 0x29, 0x41, 0x91, 0xb7, 0xdd,
-	0x99, 0xd9, 0xd9, 0xd9, 0xf9, 0xf3, 0x9b, 0x21, 0xa1, 0x77, 0xcc, 0x22, 0xca, 0xb7, 0x53, 0x9e,
-	0xc8, 0x84, 0x74, 0x71, 0xe3, 0xa5, 0x47, 0xce, 0x97, 0x70, 0xef, 0x75, 0x92, 0x9c, 0x64, 0xe9,
-	0x4b, 0xc6, 0x69, 0x20, 0x13, 0x7e, 0xb1, 0x17, 0x4b, 0x7e, 0xe1, 0xd2, 0xaf, 0x33, 0x2a, 0x24,
-	0xd9, 0x00, 0x3b, 0x2c, 0x18, 0x23, 0x6b, 0xcb, 0x7a, 0x6c, 0xbb, 0x73, 0x02, 0x21, 0xb0, 0x12,
-	0xfb, 0x33, 0x3a, 0x6a, 0x20, 0x03, 0xd7, 0xce, 0x1e, 0x6c, 0x2c, 0x57, 0x28, 0xd2, 0x24, 0x16,
-	0x94, 0x3c, 0x82, 0x16, 0x55, 0x04, 0xd4, 0xd6, 0xdb, 0xb9, 0xb5, 0x5d, 0x98, 0xb2, 0xad, 0xe5,
+	0xc5, 0xa0, 0xec, 0x22, 0x45, 0x81, 0x12, 0x14, 0xb9, 0xba, 0xdb, 0x8a, 0x47, 0xb2, 0xbb, 0x4b,
+	0xfd, 0xc9, 0x53, 0xfb, 0x35, 0x0a, 0xf4, 0xa1, 0xdf, 0xa1, 0x8f, 0x45, 0x5f, 0x8a, 0x02, 0x05,
+	0xfa, 0x2d, 0xfa, 0x49, 0x8a, 0x9d, 0x25, 0x79, 0xcb, 0xe3, 0x49, 0x4a, 0x50, 0xe4, 0x6d, 0x77,
+	0x66, 0x76, 0x76, 0x76, 0xfe, 0xfc, 0x66, 0x48, 0xe8, 0x9d, 0xb0, 0x88, 0xf2, 0x9d, 0x94, 0x27,
+	0x32, 0x21, 0x5d, 0xdc, 0x78, 0xe9, 0xb1, 0xf3, 0x15, 0xdc, 0x7b, 0x9d, 0x24, 0xa7, 0x59, 0xfa,
+	0x92, 0x71, 0x1a, 0xc8, 0x84, 0x5f, 0xee, 0xc7, 0x92, 0x5f, 0xba, 0xf4, 0x0f, 0x19, 0x15, 0x92,
+	0x6c, 0x82, 0x1d, 0x16, 0x8c, 0x91, 0xb5, 0x6d, 0x3d, 0xb6, 0xdd, 0x39, 0x81, 0x10, 0x58, 0x89,
+	0xfd, 0x19, 0x1d, 0x35, 0x90, 0x81, 0x6b, 0x67, 0x1f, 0x36, 0x97, 0x2b, 0x14, 0x69, 0x12, 0x0b,
+	0x4a, 0x1e, 0x41, 0x8b, 0x2a, 0x02, 0x6a, 0xeb, 0xed, 0x7e, 0xb0, 0x53, 0x98, 0xb2, 0xa3, 0xe5,
 	0x34, 0xd7, 0xf9, 0x87, 0x05, 0xe4, 0x35, 0x13, 0x52, 0x11, 0x19, 0x15, 0xdf, 0xce, 0x9e, 0x3b,
-	0xd0, 0x4e, 0x39, 0x3d, 0x66, 0xe7, 0xb9, 0x45, 0xf9, 0x8e, 0x3c, 0x85, 0x35, 0x21, 0x7d, 0x2e,
-	0xf7, 0x79, 0x32, 0xdb, 0x67, 0x11, 0x3d, 0x50, 0x46, 0x37, 0x51, 0xa4, 0xce, 0x20, 0xdb, 0x40,
-	0x58, 0x1c, 0x44, 0x99, 0x60, 0xa7, 0xf4, 0xb0, 0xe0, 0x8e, 0x56, 0xb6, 0xac, 0xc7, 0x5d, 0x77,
-	0x09, 0x87, 0xac, 0x43, 0x2b, 0x62, 0x33, 0x26, 0x47, 0xad, 0x2d, 0xeb, 0xf1, 0xc0, 0xd5, 0x1b,
+	0xd0, 0x4e, 0x39, 0x3d, 0x61, 0x17, 0xb9, 0x45, 0xf9, 0x8e, 0x3c, 0x85, 0x35, 0x21, 0x7d, 0x2e,
+	0x0f, 0x78, 0x32, 0x3b, 0x60, 0x11, 0x3d, 0x54, 0x46, 0x37, 0x51, 0xa4, 0xce, 0x20, 0x3b, 0x40,
+	0x58, 0x1c, 0x44, 0x99, 0x60, 0x67, 0xf4, 0xa8, 0xe0, 0x8e, 0x56, 0xb6, 0xad, 0xc7, 0x5d, 0x77,
+	0x09, 0x87, 0xac, 0x43, 0x2b, 0x62, 0x33, 0x26, 0x47, 0xad, 0x6d, 0xeb, 0xf1, 0xc0, 0xd5, 0x1b,
 	0xe7, 0xe7, 0x30, 0xac, 0xd8, 0xff, 0xdd, 0x9e, 0xff, 0x97, 0x06, 0xb4, 0x90, 0x50, 0xfa, 0xd8,
-	0x9a, 0xfb, 0x98, 0x7c, 0x08, 0x7d, 0x26, 0xbc, 0xb9, 0x23, 0x1a, 0x68, 0x5b, 0x8f, 0x89, 0xd2,
-	0xe7, 0xe4, 0x09, 0xb4, 0x83, 0x69, 0x16, 0x9f, 0x88, 0x51, 0x73, 0xab, 0xf9, 0xb8, 0xb7, 0x33,
-	0x9c, 0x5f, 0xa4, 0x1e, 0xba, 0xab, 0x78, 0x6e, 0x2e, 0x42, 0x3e, 0x01, 0xf0, 0xa5, 0xe4, 0xec,
-	0x28, 0x93, 0x54, 0xe0, 0x4b, 0x7b, 0x3b, 0x23, 0xe3, 0x40, 0x26, 0xe8, 0xf3, 0x92, 0xef, 0x1a,
-	0xb2, 0xe4, 0x53, 0xe8, 0xd2, 0x73, 0x49, 0xe3, 0x90, 0x86, 0xa3, 0x16, 0x5e, 0xb4, 0xb9, 0xf0,
-	0xa2, 0xed, 0xbd, 0x9c, 0xaf, 0xdf, 0x57, 0x8a, 0x8f, 0x3f, 0x83, 0x41, 0x85, 0x45, 0x56, 0xa1,
-	0x79, 0x42, 0x8b, 0xa8, 0xaa, 0xa5, 0xf2, 0xec, 0xa9, 0x1f, 0x65, 0x3a, 0xc1, 0xfa, 0xae, 0xde,
-	0xfc, 0xac, 0xf1, 0x89, 0xe5, 0xbc, 0x04, 0x7b, 0x3f, 0x8b, 0xa2, 0xf2, 0x60, 0xc8, 0x78, 0x71,
-	0x30, 0x64, 0x7c, 0xee, 0xe5, 0xc6, 0x95, 0x5e, 0xfe, 0xbb, 0x05, 0x6b, 0x7b, 0xa7, 0x34, 0x96,
-	0x07, 0x89, 0x64, 0xc7, 0x2c, 0xf0, 0x25, 0x4b, 0x62, 0xf2, 0x14, 0xec, 0x24, 0x0a, 0xbd, 0x2b,
-	0xc3, 0xd4, 0x4d, 0xa2, 0xdc, 0xea, 0xa7, 0x60, 0xc7, 0xf4, 0xcc, 0xbb, 0xf2, 0xba, 0x6e, 0x4c,
-	0xcf, 0xb4, 0xf4, 0x43, 0x18, 0x84, 0x34, 0xa2, 0x92, 0x7a, 0x65, 0x74, 0x54, 0xe8, 0xfa, 0x9a,
-	0xb8, 0xab, 0xc3, 0xf1, 0x11, 0xdc, 0x52, 0x2a, 0x53, 0x9f, 0xd3, 0x58, 0x7a, 0xa9, 0x2f, 0xa7,
-	0x18, 0x13, 0xdb, 0x1d, 0xc4, 0xf4, 0xec, 0x2d, 0x52, 0xdf, 0xfa, 0x72, 0xea, 0xfc, 0xad, 0x01,
-	0x76, 0x19, 0x4c, 0xf2, 0x01, 0x74, 0xd4, 0xb5, 0x1e, 0x0b, 0x73, 0x4f, 0xb4, 0xd5, 0xf6, 0x55,
-	0xa8, 0xaa, 0x22, 0x39, 0x3e, 0x16, 0x54, 0xa2, 0x79, 0x4d, 0x37, 0xdf, 0xa9, 0xcc, 0x12, 0xec,
-	0x1b, 0x5d, 0x08, 0x2b, 0x2e, 0xae, 0x95, 0xc7, 0x67, 0x92, 0xcd, 0x28, 0x5e, 0xd8, 0x74, 0xf5,
-	0x86, 0x0c, 0xa1, 0x45, 0x3d, 0xe9, 0x4f, 0x30, 0xc3, 0x6d, 0x77, 0x85, 0xbe, 0xf3, 0x27, 0xe4,
-	0x07, 0x70, 0x53, 0x24, 0x19, 0x0f, 0xa8, 0x57, 0x5c, 0xdb, 0x46, 0x6e, 0x5f, 0x53, 0xf7, 0xf5,
-	0xe5, 0x0e, 0x34, 0x8f, 0x59, 0x38, 0xea, 0xa0, 0x63, 0x56, 0xab, 0x49, 0xf8, 0x2a, 0x74, 0x15,
-	0x93, 0xfc, 0x18, 0xa0, 0xd4, 0x14, 0x8e, 0xba, 0x97, 0x88, 0xda, 0x85, 0xde, 0x90, 0x6c, 0x02,
-	0x04, 0x2c, 0x9d, 0x52, 0xee, 0xa9, 0x84, 0xb1, 0x31, 0x39, 0x6c, 0x4d, 0xf9, 0x82, 0x5e, 0x28,
-	0x36, 0x13, 0xde, 0xe4, 0x1b, 0x96, 0xa6, 0x34, 0x1c, 0x01, 0x7a, 0xd8, 0x66, 0xe2, 0x73, 0x4d,
-	0x70, 0xbe, 0x82, 0x76, 0x6e, 0xdc, 0x3d, 0xb0, 0x4f, 0x93, 0x28, 0x9b, 0x95, 0x4e, 0x1b, 0xb8,
-	0x5d, 0x4d, 0x78, 0x15, 0x92, 0xbb, 0x80, 0x28, 0x89, 0x57, 0x34, 0xd0, 0x45, 0xe8, 0x5f, 0x75,
-	0xc1, 0x1d, 0x68, 0x07, 0x49, 0x72, 0xc2, 0xb4, 0xef, 0x3a, 0x6e, 0xbe, 0x73, 0xfe, 0xd8, 0x84,
-	0x9b, 0xd5, 0x62, 0x51, 0x57, 0xa0, 0x16, 0xf4, 0xb4, 0x85, 0x6a, 0x50, 0xed, 0x61, 0xc5, 0xdb,
-	0x0d, 0xd3, 0xdb, 0xc5, 0x91, 0x59, 0x12, 0xea, 0x0b, 0x06, 0xfa, 0xc8, 0x9b, 0x24, 0xa4, 0x2a,
-	0xd7, 0x33, 0x16, 0x62, 0x78, 0x06, 0xae, 0x5a, 0x2a, 0xca, 0x84, 0x85, 0x39, 0xf8, 0xa8, 0x25,
-	0x9a, 0xc7, 0x51, 0x6f, 0x5b, 0x07, 0x5c, 0xef, 0x54, 0xc0, 0x67, 0x8a, 0xda, 0xd1, 0x51, 0x54,
-	0x6b, 0xb2, 0x05, 0x3d, 0x4e, 0xd3, 0x28, 0xcf, 0x7d, 0x74, 0xbe, 0xed, 0x9a, 0x24, 0x72, 0x1f,
-	0x20, 0x48, 0xa2, 0x88, 0x06, 0x28, 0x60, 0xa3, 0x80, 0x41, 0x51, 0x79, 0x27, 0x65, 0xe4, 0x09,
-	0x1a, 0xa0, 0xab, 0x5b, 0x6e, 0x5b, 0xca, 0xe8, 0x90, 0x06, 0xea, 0x1d, 0x99, 0xa0, 0xdc, 0x43,
-	0xf8, 0xea, 0xe1, 0xb9, 0xae, 0x22, 0x20, 0xc8, 0x6e, 0x02, 0x4c, 0x78, 0x92, 0xa5, 0x9a, 0xdb,
-	0xdf, 0x6a, 0x2a, 0x24, 0x47, 0x0a, 0xb2, 0x1f, 0xc1, 0x4d, 0x71, 0x31, 0x8b, 0x58, 0x7c, 0xe2,
-	0x49, 0x9f, 0x4f, 0xa8, 0x1c, 0x0d, 0x74, 0x05, 0xe4, 0xd4, 0x77, 0x48, 0x54, 0x6f, 0x9f, 0x85,
-	0x3f, 0x1d, 0xdd, 0xc4, 0x0c, 0x50, 0x4b, 0x27, 0x05, 0xb2, 0xcb, 0xa9, 0x2f, 0xe9, 0x77, 0x68,
-	0x63, 0xdf, 0x0e, 0x2d, 0xc8, 0x6d, 0x68, 0x27, 0x1e, 0x3d, 0x0f, 0xa2, 0xbc, 0x68, 0x5b, 0xc9,
-	0xde, 0x79, 0x10, 0x39, 0x4f, 0x60, 0x58, 0xb9, 0x31, 0x07, 0xfa, 0x75, 0x68, 0x51, 0xce, 0x93,
-	0x02, 0x96, 0xf4, 0xc6, 0xf9, 0x2d, 0x90, 0xf7, 0x69, 0xf8, 0x7d, 0x98, 0xe7, 0xdc, 0x86, 0x61,
-	0x45, 0xb5, 0xb6, 0xc3, 0xf9, 0x97, 0x05, 0xe4, 0x25, 0xa2, 0xcb, 0xff, 0xd7, 0xd8, 0x55, 0xbd,
-	0xab, 0xa6, 0xa3, 0xd1, 0x2b, 0xf4, 0xa5, 0x9f, 0xb7, 0xc4, 0x3e, 0x13, 0x5a, 0xff, 0x4b, 0x5f,
-	0xfa, 0x79, 0x6b, 0xe2, 0x34, 0xc8, 0xb8, 0xea, 0x92, 0x98, 0x96, 0xd8, 0x9a, 0xdc, 0x82, 0x44,
-	0x3e, 0x86, 0x3b, 0x6c, 0x12, 0x27, 0x9c, 0xce, 0xc5, 0x3c, 0xed, 0xaa, 0x36, 0x0a, 0xaf, 0x6b,
-	0x6e, 0x79, 0x60, 0x0f, 0x3d, 0xf7, 0x04, 0x86, 0x95, 0x67, 0x5c, 0xe9, 0xe6, 0x3f, 0x5b, 0x30,
-	0x7a, 0x2e, 0x93, 0x19, 0x0b, 0x5c, 0xaa, 0x8c, 0xaf, 0x3c, 0xfd, 0x21, 0x0c, 0x14, 0xbe, 0x2f,
-	0x3e, 0xbf, 0x9f, 0x44, 0xe1, 0xbc, 0x7f, 0xde, 0x05, 0x05, 0xf1, 0x9e, 0xe1, 0x85, 0x4e, 0x12,
-	0x85, 0x98, 0x9b, 0x0f, 0x41, 0xe1, 0xb0, 0x71, 0x5e, 0x4f, 0x12, 0xfd, 0x98, 0x9e, 0x55, 0xce,
-	0x2b, 0x21, 0x3c, 0xaf, 0xc1, 0xbb, 0x13, 0xd3, 0x33, 0x75, 0xde, 0xb9, 0x07, 0x77, 0x97, 0xd8,
-	0x96, 0x87, 0xeb, 0xdf, 0x16, 0x0c, 0x9f, 0x0b, 0xc1, 0x26, 0xf1, 0x6f, 0x10, 0x88, 0x0a, 0xa3,
-	0xd7, 0xa1, 0x15, 0x24, 0x59, 0x2c, 0xd1, 0xd8, 0x96, 0xab, 0x37, 0x0b, 0xb5, 0xd9, 0xa8, 0xd5,
-	0xe6, 0x42, 0x75, 0x37, 0xeb, 0xd5, 0x6d, 0x54, 0xef, 0x4a, 0xa5, 0x7a, 0x1f, 0x40, 0x4f, 0x05,
-	0xd9, 0x0b, 0x68, 0x2c, 0x29, 0xcf, 0x91, 0x1f, 0x14, 0x69, 0x17, 0x29, 0x4a, 0xc0, 0xec, 0x50,
-	0x1a, 0xfc, 0x21, 0x9d, 0xb7, 0xa7, 0xff, 0x58, 0xb0, 0x5e, 0x7d, 0x4a, 0x1e, 0xb3, 0x4b, 0x3b,
-	0x95, 0x02, 0x37, 0x1e, 0xe5, 0xef, 0x50, 0x4b, 0x05, 0x13, 0x69, 0x76, 0x14, 0xb1, 0xc0, 0x53,
-	0x0c, 0x6d, 0xbf, 0xad, 0x29, 0xef, 0x79, 0x34, 0xf7, 0xca, 0x8a, 0xe9, 0x15, 0x02, 0x2b, 0x7e,
-	0x26, 0xa7, 0x45, 0xb7, 0x52, 0xeb, 0x05, 0x4f, 0xb5, 0xaf, 0xf3, 0x54, 0xa7, 0xee, 0xa9, 0x32,
-	0xd3, 0xba, 0x66, 0xa6, 0x7d, 0x0c, 0x43, 0x3d, 0xee, 0x56, 0xc3, 0xb5, 0x09, 0x50, 0x76, 0x16,
-	0x31, 0xb2, 0x34, 0xbc, 0x15, 0xad, 0x45, 0x38, 0xbf, 0x00, 0xfb, 0x75, 0xa2, 0xf5, 0x0a, 0xf2,
-	0x0c, 0xec, 0xa8, 0xd8, 0xa0, 0x68, 0x6f, 0x87, 0xcc, 0x6b, 0xbc, 0x90, 0x73, 0xe7, 0x42, 0xce,
-	0x67, 0xd0, 0x2d, 0xc8, 0x85, 0xcf, 0xac, 0xcb, 0x7c, 0xd6, 0x58, 0xf0, 0x99, 0xf3, 0x4f, 0x0b,
-	0xd6, 0xab, 0x26, 0xe7, 0x61, 0x79, 0x0f, 0x83, 0xf2, 0x0a, 0x6f, 0xe6, 0xa7, 0xb9, 0x2d, 0xcf,
-	0x4c, 0x5b, 0xea, 0xc7, 0x4a, 0x03, 0xc5, 0x1b, 0x3f, 0xd5, 0xb9, 0xdc, 0x8f, 0x0c, 0xd2, 0xf8,
-	0x1d, 0xac, 0xd5, 0x44, 0x96, 0xcc, 0x7a, 0x3f, 0x32, 0x67, 0xbd, 0xca, 0xbc, 0x5a, 0x9e, 0x36,
-	0x07, 0xc0, 0x4f, 0xe1, 0x03, 0x0d, 0x07, 0xbb, 0x65, 0x0c, 0x0b, 0xdf, 0x57, 0x43, 0x6d, 0x2d,
-	0x86, 0xda, 0x19, 0xc3, 0xa8, 0x7e, 0x34, 0x2f, 0xbf, 0x09, 0xac, 0x1d, 0x4a, 0x5f, 0x32, 0x21,
-	0x59, 0x50, 0x7e, 0x74, 0x2c, 0xe4, 0x86, 0x75, 0x5d, 0x8f, 0xac, 0xd7, 0xe1, 0x2a, 0x34, 0xa5,
-	0x2c, 0xf2, 0x57, 0x2d, 0x55, 0x14, 0x88, 0x79, 0x53, 0x1e, 0x83, 0xef, 0xe1, 0x2a, 0x95, 0x0f,
-	0x32, 0x91, 0x7e, 0xa4, 0x67, 0x90, 0x15, 0x9c, 0x41, 0x6c, 0xa4, 0xe0, 0x10, 0xa2, 0xdb, 0x74,
-	0xa8, 0xb9, 0x2d, 0x3d, 0xa1, 0x28, 0x02, 0x32, 0x37, 0x01, 0xb0, 0x54, 0x75, 0x95, 0xb5, 0xf5,
-	0x59, 0x45, 0xd9, 0x55, 0x04, 0xe7, 0x3e, 0x6c, 0x7c, 0x4e, 0xa5, 0x9a, 0xa6, 0xf8, 0x6e, 0x12,
-	0x1f, 0xb3, 0x49, 0xc6, 0x7d, 0x23, 0x14, 0xce, 0x7f, 0x2d, 0xd8, 0xbc, 0x44, 0x20, 0x7f, 0xf0,
-	0x08, 0x3a, 0x33, 0x5f, 0x48, 0xca, 0x8b, 0x2a, 0x29, 0xb6, 0x8b, 0xae, 0x68, 0x5c, 0xe7, 0x8a,
-	0x66, 0xcd, 0x15, 0xb7, 0xa1, 0x3d, 0xf3, 0xcf, 0xbd, 0xd9, 0x51, 0x3e, 0x2e, 0xb5, 0x66, 0xfe,
-	0xf9, 0x9b, 0x23, 0x44, 0x36, 0xc6, 0xbd, 0xa3, 0x2c, 0x38, 0xa1, 0x52, 0x94, 0xc8, 0xc6, 0xf8,
-	0x0b, 0x4d, 0x51, 0x8f, 0x56, 0x02, 0x5f, 0x67, 0x34, 0xa3, 0x22, 0xc7, 0x0a, 0xd5, 0x1c, 0x7f,
-	0x8d, 0x04, 0x1c, 0xaf, 0x70, 0xd6, 0x44, 0x94, 0xe8, 0xba, 0xf9, 0xce, 0xc9, 0xe0, 0x8e, 0xfa,
-	0xe2, 0xa3, 0xf1, 0x7e, 0xc2, 0xf1, 0xab, 0xa2, 0x4c, 0xa0, 0x07, 0xd0, 0x0b, 0x22, 0xa6, 0xa0,
-	0xd2, 0xf8, 0x94, 0x03, 0x4d, 0xc2, 0x96, 0x82, 0x58, 0x2a, 0xa7, 0x5e, 0xe5, 0xeb, 0x15, 0x14,
-	0xe9, 0xad, 0xfe, 0x82, 0xbd, 0x0b, 0x5d, 0xc1, 0xe2, 0x80, 0x7a, 0xb1, 0xfe, 0x64, 0x68, 0xba,
-	0x1d, 0xdc, 0x1f, 0x08, 0xe7, 0x4f, 0x16, 0xdc, 0xc6, 0x6f, 0xa1, 0xda, 0x87, 0xcc, 0xd5, 0x3d,
-	0xfe, 0x57, 0x40, 0xe8, 0x29, 0xda, 0x64, 0x9c, 0xc9, 0xab, 0xef, 0x9e, 0x31, 0x63, 0x2c, 0xaa,
-	0x75, 0xd7, 0xe8, 0x22, 0xc9, 0xf1, 0x15, 0x20, 0x4d, 0x74, 0x69, 0x0f, 0xa1, 0x25, 0x85, 0x87,
-	0x50, 0xa6, 0xec, 0x5c, 0x91, 0xe2, 0x40, 0x90, 0xa7, 0x40, 0x52, 0x9f, 0x4b, 0xa6, 0xa4, 0xd5,
-	0x44, 0xed, 0x4d, 0x7d, 0x31, 0xc5, 0xcb, 0x5a, 0xee, 0x6a, 0xc9, 0xf9, 0x82, 0x5e, 0xfc, 0xd2,
-	0x17, 0x53, 0x05, 0xe0, 0x38, 0x60, 0x34, 0x71, 0xae, 0xc3, 0xf5, 0xce, 0x5f, 0xbb, 0xd0, 0x3f,
-	0xa4, 0xfe, 0x19, 0xa5, 0x21, 0xa6, 0x13, 0x99, 0x14, 0x30, 0x56, 0xfd, 0xd1, 0x40, 0x1e, 0x2d,
-	0xe2, 0xd5, 0xd2, 0x3f, 0x1b, 0xe3, 0x8f, 0xae, 0x13, 0xcb, 0x11, 0xe1, 0x06, 0x39, 0x80, 0x9e,
-	0xf1, 0x25, 0x4f, 0x36, 0x8c, 0x83, 0xb5, 0x1f, 0x14, 0xe3, 0xcd, 0x4b, 0xb8, 0x85, 0xb6, 0x67,
-	0x16, 0x79, 0x0d, 0x3d, 0x63, 0x60, 0x34, 0xf5, 0xd5, 0x27, 0x57, 0x53, 0xdf, 0x92, 0x29, 0xd3,
-	0xb9, 0xa1, 0xb4, 0x19, 0x63, 0x9f, 0xa9, 0xad, 0x3e, 0x68, 0x9a, 0xda, 0x96, 0xcd, 0x8a, 0xa8,
-	0xcd, 0x98, 0xb2, 0x4c, 0x6d, 0xf5, 0x19, 0xd2, 0xd4, 0xb6, 0x64, 0x34, 0x73, 0x6e, 0x90, 0xdf,
-	0xc3, 0x5a, 0x6d, 0xd2, 0x21, 0xce, 0xfc, 0xd4, 0x65, 0x23, 0xda, 0xf8, 0xe1, 0x95, 0x32, 0xa5,
-	0xfe, 0x2f, 0xa1, 0x6f, 0x0e, 0x18, 0xc4, 0x30, 0x68, 0xc9, 0x0c, 0x35, 0xbe, 0x7f, 0x19, 0xdb,
-	0x54, 0x68, 0xf6, 0x38, 0x53, 0xe1, 0x92, 0x2e, 0x6f, 0x2a, 0x5c, 0xd6, 0x1a, 0x9d, 0x1b, 0xe4,
-	0x77, 0xb0, 0xba, 0xd8, 0x6b, 0xc8, 0x87, 0x8b, 0x6e, 0xab, 0xb5, 0xb0, 0xb1, 0x73, 0x95, 0x48,
-	0xa9, 0xfc, 0x15, 0xc0, 0xbc, 0x85, 0x10, 0xa3, 0x66, 0x6b, 0x2d, 0x6c, 0xbc, 0xb1, 0x9c, 0x59,
-	0xaa, 0xfa, 0x03, 0xdc, 0x5e, 0x8a, 0xd3, 0xc4, 0x28, 0x93, 0xab, 0x90, 0x7e, 0xfc, 0xc3, 0x6b,
-	0xe5, 0xca, 0xbb, 0xbe, 0x82, 0x5b, 0x0b, 0x38, 0x49, 0xb6, 0xaa, 0x55, 0x53, 0x87, 0xd0, 0xf1,
-	0x03, 0xf3, 0x77, 0xd4, 0x12, 0xb0, 0x53, 0x95, 0xf5, 0xe2, 0x3e, 0xac, 0x0a, 0x0d, 0x11, 0xc7,
-	0x62, 0x5b, 0xc3, 0xeb, 0x0b, 0x40, 0x5b, 0xde, 0xf2, 0x44, 0x26, 0x47, 0x6d, 0xfc, 0xfb, 0xf9,
-	0x93, 0xff, 0x05, 0x00, 0x00, 0xff, 0xff, 0x24, 0x99, 0xf0, 0xba, 0x0c, 0x15, 0x00, 0x00,
+	0x9a, 0xfb, 0x98, 0x7c, 0x04, 0x7d, 0x26, 0xbc, 0xb9, 0x23, 0x1a, 0x68, 0x5b, 0x8f, 0x89, 0xd2,
+	0xe7, 0xe4, 0x09, 0xb4, 0x83, 0x69, 0x16, 0x9f, 0x8a, 0x51, 0x73, 0xbb, 0xf9, 0xb8, 0xb7, 0x3b,
+	0x9c, 0x5f, 0xa4, 0x1e, 0xba, 0xa7, 0x78, 0x6e, 0x2e, 0x42, 0x3e, 0x05, 0xf0, 0xa5, 0xe4, 0xec,
+	0x38, 0x93, 0x54, 0xe0, 0x4b, 0x7b, 0xbb, 0x23, 0xe3, 0x40, 0x26, 0xe8, 0xf3, 0x92, 0xef, 0x1a,
+	0xb2, 0xe4, 0x33, 0xe8, 0xd2, 0x0b, 0x49, 0xe3, 0x90, 0x86, 0xa3, 0x16, 0x5e, 0xb4, 0xb5, 0xf0,
+	0xa2, 0x9d, 0xfd, 0x9c, 0xaf, 0xdf, 0x57, 0x8a, 0x8f, 0x3f, 0x87, 0x41, 0x85, 0x45, 0x56, 0xa1,
+	0x79, 0x4a, 0x8b, 0xa8, 0xaa, 0xa5, 0xf2, 0xec, 0x99, 0x1f, 0x65, 0x3a, 0xc1, 0xfa, 0xae, 0xde,
+	0xfc, 0xac, 0xf1, 0xa9, 0xe5, 0xbc, 0x04, 0xfb, 0x20, 0x8b, 0xa2, 0xf2, 0x60, 0xc8, 0x78, 0x71,
+	0x30, 0x64, 0x7c, 0xee, 0xe5, 0xc6, 0xb5, 0x5e, 0xfe, 0xbb, 0x05, 0x6b, 0xfb, 0x67, 0x34, 0x96,
+	0x87, 0x89, 0x64, 0x27, 0x2c, 0xf0, 0x25, 0x4b, 0x62, 0xf2, 0x14, 0xec, 0x24, 0x0a, 0xbd, 0x6b,
+	0xc3, 0xd4, 0x4d, 0xa2, 0xdc, 0xea, 0xa7, 0x60, 0xc7, 0xf4, 0xdc, 0xbb, 0xf6, 0xba, 0x6e, 0x4c,
+	0xcf, 0xb5, 0xf4, 0x43, 0x18, 0x84, 0x34, 0xa2, 0x92, 0x7a, 0x65, 0x74, 0x54, 0xe8, 0xfa, 0x9a,
+	0xb8, 0xa7, 0xc3, 0xf1, 0x31, 0x7c, 0xa0, 0x54, 0xa6, 0x3e, 0xa7, 0xb1, 0xf4, 0x52, 0x5f, 0x4e,
+	0x31, 0x26, 0xb6, 0x3b, 0x88, 0xe9, 0xf9, 0x5b, 0xa4, 0xbe, 0xf5, 0xe5, 0xd4, 0xf9, 0x5b, 0x03,
+	0xec, 0x32, 0x98, 0xe4, 0x43, 0xe8, 0xa8, 0x6b, 0x3d, 0x16, 0xe6, 0x9e, 0x68, 0xab, 0xed, 0xab,
+	0x50, 0x55, 0x45, 0x72, 0x72, 0x22, 0xa8, 0x44, 0xf3, 0x9a, 0x6e, 0xbe, 0x53, 0x99, 0x25, 0xd8,
+	0x37, 0xba, 0x10, 0x56, 0x5c, 0x5c, 0x2b, 0x8f, 0xcf, 0x24, 0x9b, 0x51, 0xbc, 0xb0, 0xe9, 0xea,
+	0x0d, 0x19, 0x42, 0x8b, 0x7a, 0xd2, 0x9f, 0x60, 0x86, 0xdb, 0xee, 0x0a, 0x7d, 0xe7, 0x4f, 0xc8,
+	0x0f, 0xe0, 0xb6, 0x48, 0x32, 0x1e, 0x50, 0xaf, 0xb8, 0xb6, 0x8d, 0xdc, 0xbe, 0xa6, 0x1e, 0xe8,
+	0xcb, 0x1d, 0x68, 0x9e, 0xb0, 0x70, 0xd4, 0x41, 0xc7, 0xac, 0x56, 0x93, 0xf0, 0x55, 0xe8, 0x2a,
+	0x26, 0xf9, 0x31, 0x40, 0xa9, 0x29, 0x1c, 0x75, 0xaf, 0x10, 0xb5, 0x0b, 0xbd, 0x21, 0xd9, 0x02,
+	0x08, 0x58, 0x3a, 0xa5, 0xdc, 0x53, 0x09, 0x63, 0x63, 0x72, 0xd8, 0x9a, 0xf2, 0x25, 0xbd, 0x54,
+	0x6c, 0x26, 0xbc, 0xc9, 0x37, 0x2c, 0x4d, 0x69, 0x38, 0x02, 0xf4, 0xb0, 0xcd, 0xc4, 0x17, 0x9a,
+	0xe0, 0x7c, 0x0d, 0xed, 0xdc, 0xb8, 0x7b, 0x60, 0x9f, 0x25, 0x51, 0x36, 0x2b, 0x9d, 0x36, 0x70,
+	0xbb, 0x9a, 0xf0, 0x2a, 0x24, 0x77, 0x01, 0x51, 0x12, 0xaf, 0x68, 0xa0, 0x8b, 0xd0, 0xbf, 0xea,
+	0x82, 0x3b, 0xd0, 0x0e, 0x92, 0xe4, 0x94, 0x69, 0xdf, 0x75, 0xdc, 0x7c, 0xe7, 0xfc, 0xb1, 0x09,
+	0xb7, 0xab, 0xc5, 0xa2, 0xae, 0x40, 0x2d, 0xe8, 0x69, 0x0b, 0xd5, 0xa0, 0xda, 0xa3, 0x8a, 0xb7,
+	0x1b, 0xa6, 0xb7, 0x8b, 0x23, 0xb3, 0x24, 0xd4, 0x17, 0x0c, 0xf4, 0x91, 0x37, 0x49, 0x48, 0x55,
+	0xae, 0x67, 0x2c, 0xc4, 0xf0, 0x0c, 0x5c, 0xb5, 0x54, 0x94, 0x09, 0x0b, 0x73, 0xf0, 0x51, 0x4b,
+	0x34, 0x8f, 0xa3, 0xde, 0xb6, 0x0e, 0xb8, 0xde, 0xa9, 0x80, 0xcf, 0x14, 0xb5, 0xa3, 0xa3, 0xa8,
+	0xd6, 0x64, 0x1b, 0x7a, 0x9c, 0xa6, 0x51, 0x9e, 0xfb, 0xe8, 0x7c, 0xdb, 0x35, 0x49, 0xe4, 0x3e,
+	0x40, 0x90, 0x44, 0x11, 0x0d, 0x50, 0xc0, 0x46, 0x01, 0x83, 0xa2, 0xf2, 0x4e, 0xca, 0xc8, 0x13,
+	0x34, 0x40, 0x57, 0xb7, 0xdc, 0xb6, 0x94, 0xd1, 0x11, 0x0d, 0xd4, 0x3b, 0x32, 0x41, 0xb9, 0x87,
+	0xf0, 0xd5, 0xc3, 0x73, 0x5d, 0x45, 0x40, 0x90, 0xdd, 0x02, 0x98, 0xf0, 0x24, 0x4b, 0x35, 0xb7,
+	0xbf, 0xdd, 0x54, 0x48, 0x8e, 0x14, 0x64, 0x3f, 0x82, 0xdb, 0xe2, 0x72, 0x16, 0xb1, 0xf8, 0xd4,
+	0x93, 0x3e, 0x9f, 0x50, 0x39, 0x1a, 0xe8, 0x0a, 0xc8, 0xa9, 0xef, 0x90, 0xa8, 0xde, 0x3e, 0x0b,
+	0x7f, 0x3a, 0xba, 0x8d, 0x19, 0xa0, 0x96, 0x4e, 0x0a, 0x64, 0x8f, 0x53, 0x5f, 0xd2, 0xef, 0xd0,
+	0xc6, 0xbe, 0x1d, 0x5a, 0x90, 0x0d, 0x68, 0x27, 0x1e, 0xbd, 0x08, 0xa2, 0xbc, 0x68, 0x5b, 0xc9,
+	0xfe, 0x45, 0x10, 0x39, 0x4f, 0x60, 0x58, 0xb9, 0x31, 0x07, 0xfa, 0x75, 0x68, 0x51, 0xce, 0x93,
+	0x02, 0x96, 0xf4, 0xc6, 0xf9, 0x0d, 0x90, 0xf7, 0x69, 0xf8, 0x7d, 0x98, 0xe7, 0x6c, 0xc0, 0xb0,
+	0xa2, 0x5a, 0xdb, 0xe1, 0xfc, 0xcb, 0x02, 0xf2, 0x12, 0xd1, 0xe5, 0xff, 0x6b, 0xec, 0xaa, 0xde,
+	0x55, 0xd3, 0xd1, 0xe8, 0x15, 0xfa, 0xd2, 0xcf, 0x5b, 0x62, 0x9f, 0x09, 0xad, 0xff, 0xa5, 0x2f,
+	0xfd, 0xbc, 0x35, 0x71, 0x1a, 0x64, 0x5c, 0x75, 0x49, 0x4c, 0x4b, 0x6c, 0x4d, 0x6e, 0x41, 0x22,
+	0x9f, 0xc0, 0x1d, 0x36, 0x89, 0x13, 0x4e, 0xe7, 0x62, 0x9e, 0x76, 0x55, 0x1b, 0x85, 0xd7, 0x35,
+	0xb7, 0x3c, 0xb0, 0x8f, 0x9e, 0x7b, 0x02, 0xc3, 0xca, 0x33, 0xae, 0x75, 0xf3, 0x9f, 0x2d, 0x18,
+	0x3d, 0x97, 0xc9, 0x8c, 0x05, 0x2e, 0x55, 0xc6, 0x57, 0x9e, 0xfe, 0x10, 0x06, 0x0a, 0xdf, 0x17,
+	0x9f, 0xdf, 0x4f, 0xa2, 0x70, 0xde, 0x3f, 0xef, 0x82, 0x82, 0x78, 0xcf, 0xf0, 0x42, 0x27, 0x89,
+	0x42, 0xcc, 0xcd, 0x87, 0xa0, 0x70, 0xd8, 0x38, 0xaf, 0x27, 0x89, 0x7e, 0x4c, 0xcf, 0x2b, 0xe7,
+	0x95, 0x10, 0x9e, 0xd7, 0xe0, 0xdd, 0x89, 0xe9, 0xb9, 0x3a, 0xef, 0xdc, 0x83, 0xbb, 0x4b, 0x6c,
+	0xcb, 0xc3, 0xf5, 0x6f, 0x0b, 0x86, 0xcf, 0x85, 0x60, 0x93, 0xf8, 0xd7, 0x08, 0x44, 0x85, 0xd1,
+	0xeb, 0xd0, 0x0a, 0x92, 0x2c, 0x96, 0x68, 0x6c, 0xcb, 0xd5, 0x9b, 0x85, 0xda, 0x6c, 0xd4, 0x6a,
+	0x73, 0xa1, 0xba, 0x9b, 0xf5, 0xea, 0x36, 0xaa, 0x77, 0xa5, 0x52, 0xbd, 0x0f, 0xa0, 0xa7, 0x82,
+	0xec, 0x05, 0x34, 0x96, 0x94, 0xe7, 0xc8, 0x0f, 0x8a, 0xb4, 0x87, 0x14, 0x25, 0x60, 0x76, 0x28,
+	0x0d, 0xfe, 0x90, 0xce, 0xdb, 0xd3, 0x7f, 0x2d, 0x58, 0xaf, 0x3e, 0x25, 0x8f, 0xd9, 0x95, 0x9d,
+	0x4a, 0x81, 0x1b, 0x8f, 0xf2, 0x77, 0xa8, 0xa5, 0x82, 0x89, 0x34, 0x3b, 0x8e, 0x58, 0xe0, 0x29,
+	0x86, 0xb6, 0xdf, 0xd6, 0x94, 0xf7, 0x3c, 0x9a, 0x7b, 0x65, 0xc5, 0xf4, 0x0a, 0x81, 0x15, 0x3f,
+	0x93, 0xd3, 0xa2, 0x5b, 0xa9, 0xf5, 0x82, 0xa7, 0xda, 0x37, 0x79, 0xaa, 0x53, 0xf7, 0x54, 0x99,
+	0x69, 0x5d, 0x33, 0xd3, 0x3e, 0x81, 0xa1, 0x1e, 0x77, 0xab, 0xe1, 0xda, 0x02, 0x28, 0x3b, 0x8b,
+	0x18, 0x59, 0x1a, 0xde, 0x8a, 0xd6, 0x22, 0x9c, 0x5f, 0x80, 0xfd, 0x3a, 0xd1, 0x7a, 0x05, 0x79,
+	0x06, 0x76, 0x54, 0x6c, 0x50, 0xb4, 0xb7, 0x4b, 0xe6, 0x35, 0x5e, 0xc8, 0xb9, 0x73, 0x21, 0xe7,
+	0x73, 0xe8, 0x16, 0xe4, 0xc2, 0x67, 0xd6, 0x55, 0x3e, 0x6b, 0x2c, 0xf8, 0xcc, 0xf9, 0xa7, 0x05,
+	0xeb, 0x55, 0x93, 0xf3, 0xb0, 0xbc, 0x87, 0x41, 0x79, 0x85, 0x37, 0xf3, 0xd3, 0xdc, 0x96, 0x67,
+	0xa6, 0x2d, 0xf5, 0x63, 0xa5, 0x81, 0xe2, 0x8d, 0x9f, 0xea, 0x5c, 0xee, 0x47, 0x06, 0x69, 0xfc,
+	0x0e, 0xd6, 0x6a, 0x22, 0x4b, 0x66, 0xbd, 0x1f, 0x99, 0xb3, 0x5e, 0x65, 0x5e, 0x2d, 0x4f, 0x9b,
+	0x03, 0xe0, 0x67, 0xf0, 0xa1, 0x86, 0x83, 0xbd, 0x32, 0x86, 0x85, 0xef, 0xab, 0xa1, 0xb6, 0x16,
+	0x43, 0xed, 0x8c, 0x61, 0x54, 0x3f, 0x9a, 0x97, 0xdf, 0x04, 0xd6, 0x8e, 0xa4, 0x2f, 0x99, 0x90,
+	0x2c, 0x28, 0x3f, 0x3a, 0x16, 0x72, 0xc3, 0xba, 0xa9, 0x47, 0xd6, 0xeb, 0x70, 0x15, 0x9a, 0x52,
+	0x16, 0xf9, 0xab, 0x96, 0x2a, 0x0a, 0xc4, 0xbc, 0x29, 0x8f, 0xc1, 0xf7, 0x70, 0x95, 0xca, 0x07,
+	0x99, 0x48, 0x3f, 0xd2, 0x33, 0xc8, 0x0a, 0xce, 0x20, 0x36, 0x52, 0x70, 0x08, 0xd1, 0x6d, 0x3a,
+	0xd4, 0xdc, 0x96, 0x9e, 0x50, 0x14, 0x01, 0x99, 0x5b, 0x00, 0x58, 0xaa, 0xba, 0xca, 0xda, 0xfa,
+	0xac, 0xa2, 0xec, 0x29, 0x82, 0x73, 0x1f, 0x36, 0xbf, 0xa0, 0x52, 0x4d, 0x53, 0x7c, 0x2f, 0x89,
+	0x4f, 0xd8, 0x24, 0xe3, 0xbe, 0x11, 0x0a, 0xe7, 0x3f, 0x16, 0x6c, 0x5d, 0x21, 0x90, 0x3f, 0x78,
+	0x04, 0x9d, 0x99, 0x2f, 0x24, 0xe5, 0x45, 0x95, 0x14, 0xdb, 0x45, 0x57, 0x34, 0x6e, 0x72, 0x45,
+	0xb3, 0xe6, 0x8a, 0x0d, 0x68, 0xcf, 0xfc, 0x0b, 0x6f, 0x76, 0x9c, 0x8f, 0x4b, 0xad, 0x99, 0x7f,
+	0xf1, 0xe6, 0x18, 0x91, 0x8d, 0x71, 0xef, 0x38, 0x0b, 0x4e, 0xa9, 0x14, 0x25, 0xb2, 0x31, 0xfe,
+	0x42, 0x53, 0x70, 0x7e, 0xc2, 0x61, 0x12, 0x61, 0xa0, 0xeb, 0xe6, 0x3b, 0x27, 0x83, 0x3b, 0xea,
+	0x93, 0x8e, 0xc6, 0x07, 0x09, 0xc7, 0xcf, 0x86, 0x32, 0x43, 0x1e, 0x40, 0x2f, 0x88, 0x98, 0xc2,
+	0x42, 0xe3, 0x5b, 0x0d, 0x34, 0x09, 0x7b, 0x06, 0x82, 0xa5, 0x9c, 0x7a, 0x95, 0xcf, 0x53, 0x50,
+	0xa4, 0xb7, 0xfa, 0x13, 0xf5, 0x2e, 0x74, 0x05, 0x8b, 0x03, 0xea, 0xc5, 0xfa, 0x9b, 0xa0, 0xe9,
+	0x76, 0x70, 0x7f, 0x28, 0x9c, 0x3f, 0x59, 0xb0, 0x81, 0x1f, 0x3b, 0xb5, 0x2f, 0x95, 0xeb, 0x9b,
+	0xf8, 0xaf, 0x80, 0xd0, 0x33, 0xb4, 0xc9, 0x38, 0x93, 0x97, 0xd7, 0x3d, 0x63, 0x88, 0x58, 0x54,
+	0xeb, 0xae, 0xd1, 0x45, 0x92, 0xe3, 0x2b, 0xc4, 0x99, 0xe8, 0xda, 0x1d, 0x42, 0x4b, 0x0a, 0x0f,
+	0xb1, 0x4a, 0xd9, 0xb9, 0x22, 0xc5, 0xa1, 0x20, 0x4f, 0x81, 0xa4, 0x3e, 0x97, 0x4c, 0x49, 0xab,
+	0x91, 0xd9, 0x9b, 0xfa, 0x62, 0x8a, 0x97, 0xb5, 0xdc, 0xd5, 0x92, 0xf3, 0x25, 0xbd, 0xfc, 0xa5,
+	0x2f, 0xa6, 0x0a, 0xa1, 0x71, 0x82, 0x68, 0xe2, 0xe0, 0x86, 0xeb, 0xdd, 0xbf, 0x76, 0xa1, 0x7f,
+	0x44, 0xfd, 0x73, 0x4a, 0x43, 0xcc, 0x17, 0x32, 0x29, 0x70, 0xaa, 0xfa, 0x27, 0x81, 0x3c, 0x5a,
+	0x04, 0xa4, 0xa5, 0xbf, 0x2e, 0xc6, 0x1f, 0xdf, 0x24, 0x96, 0x97, 0xfc, 0x2d, 0x72, 0x08, 0x3d,
+	0xe3, 0x53, 0x9d, 0x6c, 0x1a, 0x07, 0x6b, 0x7f, 0x20, 0xc6, 0x5b, 0x57, 0x70, 0x0b, 0x6d, 0xcf,
+	0x2c, 0xf2, 0x1a, 0x7a, 0xc6, 0x44, 0x68, 0xea, 0xab, 0x8f, 0xa6, 0xa6, 0xbe, 0x25, 0x63, 0xa4,
+	0x73, 0x4b, 0x69, 0x33, 0xe6, 0x3a, 0x53, 0x5b, 0x7d, 0x92, 0x34, 0xb5, 0x2d, 0x1b, 0x06, 0x51,
+	0x9b, 0x31, 0x46, 0x99, 0xda, 0xea, 0x43, 0xa2, 0xa9, 0x6d, 0xc9, 0xec, 0xe5, 0xdc, 0x22, 0xbf,
+	0x83, 0xb5, 0xda, 0x28, 0x43, 0x9c, 0xf9, 0xa9, 0xab, 0x66, 0xb0, 0xf1, 0xc3, 0x6b, 0x65, 0x4a,
+	0xfd, 0x5f, 0x41, 0xdf, 0x9c, 0x20, 0x88, 0x61, 0xd0, 0x92, 0x21, 0x69, 0x7c, 0xff, 0x2a, 0xb6,
+	0xa9, 0xd0, 0x6c, 0x62, 0xa6, 0xc2, 0x25, 0x6d, 0xdc, 0x54, 0xb8, 0xac, 0xf7, 0x39, 0xb7, 0xc8,
+	0x6f, 0x61, 0x75, 0xb1, 0x99, 0x90, 0x8f, 0x16, 0xdd, 0x56, 0xeb, 0x51, 0x63, 0xe7, 0x3a, 0x91,
+	0x52, 0xf9, 0x2b, 0x80, 0x79, 0x8f, 0x20, 0x46, 0xcd, 0xd6, 0x7a, 0xd4, 0x78, 0x73, 0x39, 0xb3,
+	0x54, 0xf5, 0x7b, 0xd8, 0x58, 0x0a, 0xc4, 0xc4, 0x28, 0x93, 0xeb, 0xa0, 0x7c, 0xfc, 0xc3, 0x1b,
+	0xe5, 0xca, 0xbb, 0xbe, 0x86, 0x0f, 0x16, 0x70, 0x92, 0x6c, 0x57, 0xab, 0xa6, 0x0e, 0xa1, 0xe3,
+	0x07, 0xe6, 0xff, 0xa6, 0x25, 0x60, 0xa7, 0x2a, 0xeb, 0xc5, 0x7d, 0x58, 0x15, 0x1a, 0x22, 0x4e,
+	0xc4, 0x8e, 0x86, 0xd7, 0x17, 0x80, 0xb6, 0xbc, 0xe5, 0x89, 0x4c, 0x8e, 0xdb, 0xf8, 0x7b, 0xf3,
+	0x27, 0xff, 0x0b, 0x00, 0x00, 0xff, 0xff, 0x0d, 0xae, 0xfb, 0x80, 0xed, 0x14, 0x00, 0x00,
 }
diff --git a/weed/server/filer_grpc_server.go b/weed/server/filer_grpc_server.go
index ead246eb8..549553acc 100644
--- a/weed/server/filer_grpc_server.go
+++ b/weed/server/filer_grpc_server.go
@@ -327,7 +327,6 @@ func (fs *FilerServer) GetFilerConfiguration(ctx context.Context, req *filer_pb.
 		Replication: fs.option.DefaultReplication,
 		MaxMb:       uint32(fs.option.MaxMB),
 		DirBuckets:  fs.filer.DirBucketsPath,
-		DirQueues:   fs.filer.DirQueuesPath,
 		Cipher:      fs.filer.Cipher,
 	}, nil
 }
diff --git a/weed/server/filer_server.go b/weed/server/filer_server.go
index 63bc8bd03..d8761a538 100644
--- a/weed/server/filer_server.go
+++ b/weed/server/filer_server.go
@@ -93,9 +93,7 @@ func NewFilerServer(defaultMux, readonlyMux *http.ServeMux, option *FilerOption)
 
 	fs.option.recursiveDelete = v.GetBool("filer.options.recursive_delete")
 	v.SetDefault("filer.options.buckets_folder", "/buckets")
-	v.SetDefault("filer.options.queues_folder", "/queues")
 	fs.filer.DirBucketsPath = v.GetString("filer.options.buckets_folder")
-	fs.filer.DirQueuesPath = v.GetString("filer.options.queues_folder")
 	fs.filer.FsyncBuckets = v.GetStringSlice("filer.options.buckets_fsync")
 	fs.filer.LoadConfiguration(v)
 

From d30483d642b6115d32a7d373a33538da25e45f28 Mon Sep 17 00:00:00 2001
From: Chris Lu <chris.lu@gmail.com>
Date: Sun, 12 Apr 2020 14:03:07 -0700
Subject: [PATCH 51/81] re-enable system logs

---
 unmaintained/see_log_entry/see_log_entry.go |  3 +-
 weed/filer2/filer.go                        |  6 ++-
 weed/filer2/filer_notify.go                 |  9 ++--
 weed/filer2/filer_notify_append.go          | 47 +++++++++++++--------
 weed/filer2/leveldb/leveldb_store_test.go   |  4 +-
 weed/filer2/leveldb2/leveldb2_store_test.go |  4 +-
 weed/filer2/topics.go                       |  6 +++
 weed/server/filer_grpc_server_listen.go     |  3 +-
 weed/server/filer_server.go                 |  2 +-
 9 files changed, 53 insertions(+), 31 deletions(-)
 create mode 100644 weed/filer2/topics.go

diff --git a/unmaintained/see_log_entry/see_log_entry.go b/unmaintained/see_log_entry/see_log_entry.go
index b7d724344..69503b65d 100644
--- a/unmaintained/see_log_entry/see_log_entry.go
+++ b/unmaintained/see_log_entry/see_log_entry.go
@@ -9,12 +9,13 @@ import (
 
 	"github.com/golang/protobuf/proto"
 
+	"github.com/chrislusf/seaweedfs/weed/filer2"
 	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
 	"github.com/chrislusf/seaweedfs/weed/util"
 )
 
 var (
-	logdataFile = flag.String("logdata", "", "log data file saved under /.meta/log/...")
+	logdataFile = flag.String("logdata", "", "log data file saved under "+ filer2.SystemLogDir)
 )
 
 func main() {
diff --git a/weed/filer2/filer.go b/weed/filer2/filer.go
index df6379d26..acd609847 100644
--- a/weed/filer2/filer.go
+++ b/weed/filer2/filer.go
@@ -36,9 +36,11 @@ type Filer struct {
 	buckets             *FilerBuckets
 	Cipher              bool
 	metaLogBuffer       *log_buffer.LogBuffer
+	metaLogCollection   string
+	metaLogReplication  string
 }
 
-func NewFiler(masters []string, grpcDialOption grpc.DialOption, filerGrpcPort uint32, notifyFn func()) *Filer {
+func NewFiler(masters []string, grpcDialOption grpc.DialOption, filerGrpcPort uint32, collection string, replication string, notifyFn func()) *Filer {
 	f := &Filer{
 		directoryCache:      ccache.New(ccache.Configure().MaxSize(1000).ItemsToPrune(100)),
 		MasterClient:        wdclient.NewMasterClient(grpcDialOption, "filer", filerGrpcPort, masters),
@@ -46,6 +48,8 @@ func NewFiler(masters []string, grpcDialOption grpc.DialOption, filerGrpcPort ui
 		GrpcDialOption:      grpcDialOption,
 	}
 	f.metaLogBuffer = log_buffer.NewLogBuffer(time.Minute, f.logFlushFunc, notifyFn)
+	f.metaLogCollection = collection
+	f.metaLogReplication = replication
 
 	go f.loopProcessingDeletion()
 
diff --git a/weed/filer2/filer_notify.go b/weed/filer2/filer_notify.go
index de07e1cf9..18f96658b 100644
--- a/weed/filer2/filer_notify.go
+++ b/weed/filer2/filer_notify.go
@@ -25,7 +25,7 @@ func (f *Filer) NotifyUpdateEvent(oldEntry, newEntry *Entry, deleteChunks bool)
 
 	// println("fullpath:", fullpath)
 
-	if strings.HasPrefix(fullpath, "/.meta") {
+	if strings.HasPrefix(fullpath, SystemLogDir) {
 		return
 	}
 
@@ -69,11 +69,10 @@ func (f *Filer) logMetaEvent(fullpath string, eventNotification *filer_pb.EventN
 
 func (f *Filer) logFlushFunc(startTime, stopTime time.Time, buf []byte) {
 
-	return
-
-	targetFile := fmt.Sprintf("/.meta/log/%04d/%02d/%02d/%02d/%02d/%02d.%09d.log",
+	targetFile := fmt.Sprintf("%s/%04d-%02d-%02d/%02d-%02d.segment", SystemLogDir,
 		startTime.Year(), startTime.Month(), startTime.Day(), startTime.Hour(), startTime.Minute(),
-		startTime.Second(), startTime.Nanosecond())
+		// startTime.Second(), startTime.Nanosecond(),
+	)
 
 	if err := f.appendToFile(targetFile, buf); err != nil {
 		glog.V(0).Infof("log write failed %s: %v", targetFile, err)
diff --git a/weed/filer2/filer_notify_append.go b/weed/filer2/filer_notify_append.go
index 4c134ae66..0e6e8d50f 100644
--- a/weed/filer2/filer_notify_append.go
+++ b/weed/filer2/filer_notify_append.go
@@ -13,25 +13,10 @@ import (
 
 func (f *Filer) appendToFile(targetFile string, data []byte) error {
 
-	// assign a volume location
-	assignRequest := &operation.VolumeAssignRequest{
-		Count: 1,
-	}
-	assignResult, err := operation.Assign(f.GetMaster(), f.GrpcDialOption, assignRequest)
-	if err != nil {
-		return fmt.Errorf("AssignVolume: %v", err)
-	}
-	if assignResult.Error != "" {
-		return fmt.Errorf("AssignVolume error: %v", assignResult.Error)
-	}
-
-	// upload data
-	targetUrl := "http://" + assignResult.Url + "/" + assignResult.Fid
-	uploadResult, err := operation.UploadData(targetUrl, "", false, data, false, "", nil, assignResult.Auth)
-	if err != nil {
-		return fmt.Errorf("upload data %s: %v", targetUrl, err)
+	assignResult, err, uploadResult, err2 := f.assignAndUpload(data)
+	if err2 != nil {
+		return err2
 	}
-	// println("uploaded to", targetUrl)
 
 	// find out existing entry
 	fullpath := util.FullPath(targetFile)
@@ -68,3 +53,29 @@ func (f *Filer) appendToFile(targetFile string, data []byte) error {
 
 	return err
 }
+
+func (f *Filer) assignAndUpload(data []byte) (*operation.AssignResult, error, *operation.UploadResult, error) {
+	// assign a volume location
+	assignRequest := &operation.VolumeAssignRequest{
+		Count:               1,
+		Collection:          f.metaLogCollection,
+		Replication:         f.metaLogReplication,
+		WritableVolumeCount: 1,
+	}
+	assignResult, err := operation.Assign(f.GetMaster(), f.GrpcDialOption, assignRequest)
+	if err != nil {
+		return nil, nil, nil, fmt.Errorf("AssignVolume: %v", err)
+	}
+	if assignResult.Error != "" {
+		return nil, nil, nil, fmt.Errorf("AssignVolume error: %v", assignResult.Error)
+	}
+
+	// upload data
+	targetUrl := "http://" + assignResult.Url + "/" + assignResult.Fid
+	uploadResult, err := operation.UploadData(targetUrl, "", false, data, false, "", nil, assignResult.Auth)
+	if err != nil {
+		return nil, nil, nil, fmt.Errorf("upload data %s: %v", targetUrl, err)
+	}
+	// println("uploaded to", targetUrl)
+	return assignResult, err, uploadResult, nil
+}
diff --git a/weed/filer2/leveldb/leveldb_store_test.go b/weed/filer2/leveldb/leveldb_store_test.go
index 21d126322..4f415bb9c 100644
--- a/weed/filer2/leveldb/leveldb_store_test.go
+++ b/weed/filer2/leveldb/leveldb_store_test.go
@@ -11,7 +11,7 @@ import (
 )
 
 func TestCreateAndFind(t *testing.T) {
-	filer := filer2.NewFiler(nil, nil, 0, nil)
+	filer := filer2.NewFiler(nil, nil, 0, "", "", nil)
 	dir, _ := ioutil.TempDir("", "seaweedfs_filer_test")
 	defer os.RemoveAll(dir)
 	store := &LevelDBStore{}
@@ -66,7 +66,7 @@ func TestCreateAndFind(t *testing.T) {
 }
 
 func TestEmptyRoot(t *testing.T) {
-	filer := filer2.NewFiler(nil, nil, 0, nil)
+	filer := filer2.NewFiler(nil, nil, 0, "", "", nil)
 	dir, _ := ioutil.TempDir("", "seaweedfs_filer_test2")
 	defer os.RemoveAll(dir)
 	store := &LevelDBStore{}
diff --git a/weed/filer2/leveldb2/leveldb2_store_test.go b/weed/filer2/leveldb2/leveldb2_store_test.go
index 324b07d6c..d4ab2c163 100644
--- a/weed/filer2/leveldb2/leveldb2_store_test.go
+++ b/weed/filer2/leveldb2/leveldb2_store_test.go
@@ -11,7 +11,7 @@ import (
 )
 
 func TestCreateAndFind(t *testing.T) {
-	filer := filer2.NewFiler(nil, nil, 0, nil)
+	filer := filer2.NewFiler(nil, nil, 0, "", "", nil)
 	dir, _ := ioutil.TempDir("", "seaweedfs_filer_test")
 	defer os.RemoveAll(dir)
 	store := &LevelDB2Store{}
@@ -66,7 +66,7 @@ func TestCreateAndFind(t *testing.T) {
 }
 
 func TestEmptyRoot(t *testing.T) {
-	filer := filer2.NewFiler(nil, nil, 0, nil)
+	filer := filer2.NewFiler(nil, nil, 0, "", "", nil)
 	dir, _ := ioutil.TempDir("", "seaweedfs_filer_test2")
 	defer os.RemoveAll(dir)
 	store := &LevelDB2Store{}
diff --git a/weed/filer2/topics.go b/weed/filer2/topics.go
new file mode 100644
index 000000000..8ab409774
--- /dev/null
+++ b/weed/filer2/topics.go
@@ -0,0 +1,6 @@
+package filer2
+
+const(
+	TopicsDir = "/topics"
+	SystemLogDir = TopicsDir + "/.system/log"
+)
diff --git a/weed/server/filer_grpc_server_listen.go b/weed/server/filer_grpc_server_listen.go
index 9c98f74e5..e2d932a99 100644
--- a/weed/server/filer_grpc_server_listen.go
+++ b/weed/server/filer_grpc_server_listen.go
@@ -4,6 +4,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/chrislusf/seaweedfs/weed/filer2"
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
 	"github.com/chrislusf/seaweedfs/weed/util"
@@ -37,7 +38,7 @@ func (fs *FilerServer) ListenForEvents(req *filer_pb.ListenForEventsRequest, str
 			fullpath := util.Join(dirPath, entryName)
 
 			// skip on filer internal meta logs
-			if strings.HasPrefix(fullpath, "/.meta") {
+			if strings.HasPrefix(fullpath, filer2.SystemLogDir) {
 				return nil
 			}
 
diff --git a/weed/server/filer_server.go b/weed/server/filer_server.go
index d8761a538..96a5545f4 100644
--- a/weed/server/filer_server.go
+++ b/weed/server/filer_server.go
@@ -73,7 +73,7 @@ func NewFilerServer(defaultMux, readonlyMux *http.ServeMux, option *FilerOption)
 		glog.Fatal("master list is required!")
 	}
 
-	fs.filer = filer2.NewFiler(option.Masters, fs.grpcDialOption, option.Port+10000, fs.notifyMetaListeners)
+	fs.filer = filer2.NewFiler(option.Masters, fs.grpcDialOption, option.Port+10000, option.Collection, option.DefaultReplication, fs.notifyMetaListeners)
 	fs.filer.Cipher = option.Cipher
 
 	maybeStartMetrics(fs, option)

From eb16bb028779fc09ae9811627c300ff9f28e8948 Mon Sep 17 00:00:00 2001
From: Chris Lu <chris.lu@gmail.com>
Date: Sun, 12 Apr 2020 16:44:58 -0700
Subject: [PATCH 52/81] go mod cleanup

---
 go.mod                     | 4 +---
 go.sum                     | 9 +++------
 weed/images/orientation.go | 2 +-
 3 files changed, 5 insertions(+), 10 deletions(-)

diff --git a/go.mod b/go.mod
index e408618f3..2dee477eb 100644
--- a/go.mod
+++ b/go.mod
@@ -38,7 +38,6 @@ require (
 	github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4 // indirect
 	github.com/grpc-ecosystem/grpc-gateway v1.11.0 // indirect
 	github.com/hashicorp/golang-lru v0.5.3 // indirect
-	github.com/jacobsa/daemonize v0.0.0-20160101105449-e460293e890f
 	github.com/jcmturner/gofork v1.0.0 // indirect
 	github.com/karlseguin/ccache v2.0.3+incompatible
 	github.com/karlseguin/expect v1.0.1 // indirect
@@ -58,12 +57,11 @@ require (
 	github.com/peterh/liner v1.1.0
 	github.com/pierrec/lz4 v2.2.7+incompatible // indirect
 	github.com/prometheus/client_golang v1.1.0
-	github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 // indirect
 	github.com/prometheus/procfs v0.0.4 // indirect
 	github.com/rakyll/statik v0.1.6
 	github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563 // indirect
-	github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd
 	github.com/seaweedfs/fuse v0.0.0-20190510212405-310228904eff
+	github.com/seaweedfs/goexif v1.0.2
 	github.com/sirupsen/logrus v1.4.2 // indirect
 	github.com/spaolacci/murmur3 v1.1.0 // indirect
 	github.com/spf13/afero v1.2.2 // indirect
diff --git a/go.sum b/go.sum
index fff96e5f6..77845f263 100644
--- a/go.sum
+++ b/go.sum
@@ -59,6 +59,7 @@ github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4Yn
 github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4=
 github.com/census-instrumentation/opencensus-proto v0.2.0 h1:LzQXZOgg4CQfE6bFvXGM30YZL1WW/M337pXml+GrcZ4=
 github.com/census-instrumentation/opencensus-proto v0.2.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
+github.com/census-instrumentation/opencensus-proto v0.2.1 h1:glEXhBS5PSLLv4IXzLA5yPRVX4bilULVyxxbrfOtDAk=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
 github.com/chrislusf/raft v0.0.0-20190225081310-10d6e2182d92 h1:lM9SFsh0EPXkyJyrTJqLZPAIJBtNFP6LNkYXu2MnSZI=
@@ -129,8 +130,6 @@ github.com/go-ini/ini v1.25.4/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3I
 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
 github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
-github.com/go-redis/redis v6.15.2+incompatible h1:9SpNVG76gr6InJGxoZ6IuuxaCOQwDAhzyXg+Bs+0Sb4=
-github.com/go-redis/redis v6.15.2+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA=
 github.com/go-redis/redis v6.15.7+incompatible h1:3skhDh95XQMpnqeqNftPkQD9jL9e5e36z/1SUm6dy1U=
 github.com/go-redis/redis v6.15.7+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA=
 github.com/go-sql-driver/mysql v1.4.1 h1:g24URVg0OFbNUTx9qqY1IRZ9D9z3iPyi5zKhQZpNwpA=
@@ -219,8 +218,6 @@ github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
 github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
-github.com/jacobsa/daemonize v0.0.0-20160101105449-e460293e890f h1:X+tnaqoCcBgAwSTJtoYW6p0qKiuPyMfofEHEFUf2kdU=
-github.com/jacobsa/daemonize v0.0.0-20160101105449-e460293e890f/go.mod h1:Ip4fOwzCrnDVuluHBd7FXIMb7SHOKfkt9/UDrYSZvqI=
 github.com/jcmturner/gofork v0.0.0-20190328161633-dc7c13fece03 h1:FUwcHNlEqkqLjLBdCp5PRlCFijNjvcYANOZXzCfXwCM=
 github.com/jcmturner/gofork v0.0.0-20190328161633-dc7c13fece03/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o=
 github.com/jcmturner/gofork v1.0.0 h1:J7uCkflzTEhUZ64xqKnkDxq3kzc96ajM1Gli5ktUem8=
@@ -362,10 +359,10 @@ github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqn
 github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563 h1:dY6ETXrvDG7Sa4vE8ZQG4yqWg6UnOcbqTAahkV813vQ=
 github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
 github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
-github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd h1:CmH9+J6ZSsIjUK3dcGsnCnO41eRBOnY12zwkn5qVwgc=
-github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk=
 github.com/seaweedfs/fuse v0.0.0-20190510212405-310228904eff h1:uLd5zBvf5OA67wcVRePHrFt60bR4LSskaVhgVwyk0Jg=
 github.com/seaweedfs/fuse v0.0.0-20190510212405-310228904eff/go.mod h1:cubdLmQFqEUZ9vNJrznhgc3m3VMAJi/nY2Ix2axXkG0=
+github.com/seaweedfs/goexif v1.0.2 h1:p+rTXYdQ2mgxd+1JaTrQ9N8DvYuw9UH9xgYmJ+Bb29E=
+github.com/seaweedfs/goexif v1.0.2/go.mod h1:MrKs5LK0HXdffrdCZrW3OIMegL2xXpC6ThLyXMyjdrk=
 github.com/sirupsen/logrus v1.2.0 h1:juTguoYk5qI21pwyTXY3B3Y5cOTH3ZUyZCg1v/mihuo=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4=
diff --git a/weed/images/orientation.go b/weed/images/orientation.go
index 4bff89311..a592a7d8b 100644
--- a/weed/images/orientation.go
+++ b/weed/images/orientation.go
@@ -7,7 +7,7 @@ import (
 	"image/jpeg"
 	"log"
 
-	"github.com/rwcarlsen/goexif/exif"
+	"github.com/seaweedfs/goexif/exif"
 )
 
 //many code is copied from http://camlistore.org/pkg/images/images.go

From 295f00cdce8c76dce5c1278d8b88297b7a087424 Mon Sep 17 00:00:00 2001
From: Chris Lu <chris.lu@gmail.com>
Date: Sun, 12 Apr 2020 17:04:16 -0700
Subject: [PATCH 53/81] update imaging lib

---
 go.mod | 4 ++--
 go.sum | 5 +++++
 2 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/go.mod b/go.mod
index 2dee477eb..2d7d98b8a 100644
--- a/go.mod
+++ b/go.mod
@@ -15,7 +15,7 @@ require (
 	github.com/coreos/go-semver v0.3.0 // indirect
 	github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f // indirect
 	github.com/dgrijalva/jwt-go v3.2.0+incompatible
-	github.com/disintegration/imaging v1.6.1
+	github.com/disintegration/imaging v1.6.2
 	github.com/dustin/go-humanize v1.0.0
 	github.com/eapache/go-resiliency v1.2.0 // indirect
 	github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a
@@ -82,7 +82,7 @@ require (
 	gocloud.dev/pubsub/natspubsub v0.16.0
 	gocloud.dev/pubsub/rabbitpubsub v0.16.0
 	golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7 // indirect
-	golang.org/x/image v0.0.0-20190829233526-b3c06291d021 // indirect
+	golang.org/x/image v0.0.0-20200119044424-58c23975cae1 // indirect
 	golang.org/x/net v0.0.0-20190909003024-a7b16738d86b
 	golang.org/x/sys v0.0.0-20190910064555-bbd175535a8b
 	golang.org/x/tools v0.0.0-20190911022129-16c5e0f7d110
diff --git a/go.sum b/go.sum
index 77845f263..98dc32f16 100644
--- a/go.sum
+++ b/go.sum
@@ -93,6 +93,8 @@ github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8
 github.com/dimchansky/utfbom v1.1.0/go.mod h1:rO41eb7gLfo8SF1jd9F8HplJm1Fewwi4mQvIirEdv+8=
 github.com/disintegration/imaging v1.6.1 h1:JnBbK6ECIZb1NsWIikP9pd8gIlTIRx7fuDNpU9fsxOE=
 github.com/disintegration/imaging v1.6.1/go.mod h1:xuIt+sRxDFrHS0drzXUlCJthkJ8k7lkkUojDSR247MQ=
+github.com/disintegration/imaging v1.6.2 h1:w1LecBlG2Lnp8B3jk5zSuNqd7b4DXhcjwek1ei82L+c=
+github.com/disintegration/imaging v1.6.2/go.mod h1:44/5580QXChDfwIclfc/PCwrr44amcmDAg8hxG0Ewe4=
 github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo=
 github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
 github.com/eapache/go-resiliency v1.1.0 h1:1NtRmCAqadE2FN4ZcN6g90TP3uk8cg9rn9eNK2197aU=
@@ -462,6 +464,9 @@ golang.org/x/image v0.0.0-20190227222117-0694c2d4d067 h1:KYGJGHOQy8oSi1fDlSpcZF0
 golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
 golang.org/x/image v0.0.0-20190829233526-b3c06291d021 h1:j6QOxNFMpEL1wIQX6TUdBPNfGZKmBOJS/vfSm8a7tdM=
 golang.org/x/image v0.0.0-20190829233526-b3c06291d021/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
+golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
+golang.org/x/image v0.0.0-20200119044424-58c23975cae1 h1:5h3ngYt7+vXCDZCup/HkCQgW5XwmSvR/nA2JmJ0RErg=
+golang.org/x/image v0.0.0-20200119044424-58c23975cae1/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
 golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=

From 65919d971d354aac254aaf9b1b064b038e7b37fe Mon Sep 17 00:00:00 2001
From: Chris Lu <chris.lu@gmail.com>
Date: Sun, 12 Apr 2020 17:25:54 -0700
Subject: [PATCH 54/81] update statik

---
 go.mod | 2 +-
 go.sum | 2 ++
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/go.mod b/go.mod
index 2d7d98b8a..7f243404c 100644
--- a/go.mod
+++ b/go.mod
@@ -58,7 +58,7 @@ require (
 	github.com/pierrec/lz4 v2.2.7+incompatible // indirect
 	github.com/prometheus/client_golang v1.1.0
 	github.com/prometheus/procfs v0.0.4 // indirect
-	github.com/rakyll/statik v0.1.6
+	github.com/rakyll/statik v0.1.7
 	github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563 // indirect
 	github.com/seaweedfs/fuse v0.0.0-20190510212405-310228904eff
 	github.com/seaweedfs/goexif v1.0.2
diff --git a/go.sum b/go.sum
index 98dc32f16..450328d54 100644
--- a/go.sum
+++ b/go.sum
@@ -356,6 +356,8 @@ github.com/prometheus/procfs v0.0.4/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDa
 github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
 github.com/rakyll/statik v0.1.6 h1:uICcfUXpgqtw2VopbIncslhAmE5hwc4g20TEyEENBNs=
 github.com/rakyll/statik v0.1.6/go.mod h1:OEi9wJV/fMUAGx1eNjq75DKDsJVuEv1U0oYdX6GX8Zs=
+github.com/rakyll/statik v0.1.7 h1:OF3QCZUuyPxuGEP7B4ypUa7sB/iHtqOTDYZXGM8KOdQ=
+github.com/rakyll/statik v0.1.7/go.mod h1:AlZONWzMtEnMs7W4e/1LURLiI49pIMmp6V9Unghqrcc=
 github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a h1:9ZKAASQSHhDYGoxY8uLVpewe1GDZ2vu2Tr/vTdVAkFQ=
 github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
 github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563 h1:dY6ETXrvDG7Sa4vE8ZQG4yqWg6UnOcbqTAahkV813vQ=

From 5361f999ed112aa97b786e09256839db7a59ee3d Mon Sep 17 00:00:00 2001
From: Chris Lu <chris.lu@gmail.com>
Date: Sun, 12 Apr 2020 17:48:39 -0700
Subject: [PATCH 55/81] shell: only one shell is allowed to connect to the
 cluster

fix https://github.com/chrislusf/seaweedfs/issues/1266
---
 weed/server/master_grpc_server.go | 13 +++++++++++++
 weed/server/master_server.go      |  2 ++
 2 files changed, 15 insertions(+)

diff --git a/weed/server/master_grpc_server.go b/weed/server/master_grpc_server.go
index e5fcacc0e..1785a8ccb 100644
--- a/weed/server/master_grpc_server.go
+++ b/weed/server/master_grpc_server.go
@@ -190,6 +190,19 @@ func (ms *MasterServer) KeepConnected(stream master_pb.Seaweed_KeepConnectedServ
 
 	peerAddress := findClientAddress(stream.Context(), req.GrpcPort)
 
+	// only one shell can be connected at any time
+	if req.Name == "shell" {
+		if ms.currentAdminShellClient == ""{
+			ms.currentAdminShellClient = peerAddress
+			defer func() {
+				ms.currentAdminShellClient = ""
+			}()
+		} else {
+			return fmt.Errorf("only one concurrent shell allowed, but another shell is already connected from %s", peerAddress)
+		}
+	}
+
+
 	stopChan := make(chan bool)
 
 	clientName, messageChan := ms.addClient(req.Name, peerAddress)
diff --git a/weed/server/master_server.go b/weed/server/master_server.go
index 4a264d432..d089370db 100644
--- a/weed/server/master_server.go
+++ b/weed/server/master_server.go
@@ -64,6 +64,8 @@ type MasterServer struct {
 	grpcDialOption grpc.DialOption
 
 	MasterClient *wdclient.MasterClient
+
+	currentAdminShellClient string
 }
 
 func NewMasterServer(r *mux.Router, option *MasterOption, peers []string) *MasterServer {

From b062393f3f90388ac98706ed22bd20907015b265 Mon Sep 17 00:00:00 2001
From: Chris Lu <chris.lu@gmail.com>
Date: Sun, 12 Apr 2020 17:51:31 -0700
Subject: [PATCH 56/81] refactoring

---
 weed/wdclient/masterclient.go | 26 +++++++++++++-------------
 1 file changed, 13 insertions(+), 13 deletions(-)

diff --git a/weed/wdclient/masterclient.go b/weed/wdclient/masterclient.go
index 301f20615..67e4509b9 100644
--- a/weed/wdclient/masterclient.go
+++ b/weed/wdclient/masterclient.go
@@ -13,7 +13,7 @@ import (
 )
 
 type MasterClient struct {
-	name           string
+	clientType     string
 	grpcPort       uint32
 	currentMaster  string
 	masters        []string
@@ -22,9 +22,9 @@ type MasterClient struct {
 	vidMap
 }
 
-func NewMasterClient(grpcDialOption grpc.DialOption, clientName string, clientGrpcPort uint32, masters []string) *MasterClient {
+func NewMasterClient(grpcDialOption grpc.DialOption, clientType string, clientGrpcPort uint32, masters []string) *MasterClient {
 	return &MasterClient{
-		name:           clientName,
+		clientType:     clientType,
 		grpcPort:       clientGrpcPort,
 		masters:        masters,
 		grpcDialOption: grpcDialOption,
@@ -43,7 +43,7 @@ func (mc *MasterClient) WaitUntilConnected() {
 }
 
 func (mc *MasterClient) KeepConnectedToMaster() {
-	glog.V(1).Infof("%s bootstraps with masters %v", mc.name, mc.masters)
+	glog.V(1).Infof("%s bootstraps with masters %v", mc.clientType, mc.masters)
 	for {
 		mc.tryAllMasters()
 		time.Sleep(time.Second)
@@ -65,27 +65,27 @@ func (mc *MasterClient) tryAllMasters() {
 }
 
 func (mc *MasterClient) tryConnectToMaster(master string) (nextHintedLeader string) {
-	glog.V(1).Infof("%s Connecting to master %v", mc.name, master)
+	glog.V(1).Infof("%s Connecting to master %v", mc.clientType, master)
 	gprcErr := pb.WithMasterClient(master, mc.grpcDialOption, func(client master_pb.SeaweedClient) error {
 
 		stream, err := client.KeepConnected(context.Background())
 		if err != nil {
-			glog.V(0).Infof("%s failed to keep connected to %s: %v", mc.name, master, err)
+			glog.V(0).Infof("%s failed to keep connected to %s: %v", mc.clientType, master, err)
 			return err
 		}
 
-		if err = stream.Send(&master_pb.KeepConnectedRequest{Name: mc.name, GrpcPort: mc.grpcPort}); err != nil {
-			glog.V(0).Infof("%s failed to send to %s: %v", mc.name, master, err)
+		if err = stream.Send(&master_pb.KeepConnectedRequest{Name: mc.clientType, GrpcPort: mc.grpcPort}); err != nil {
+			glog.V(0).Infof("%s failed to send to %s: %v", mc.clientType, master, err)
 			return err
 		}
 
-		glog.V(1).Infof("%s Connected to %v", mc.name, master)
+		glog.V(1).Infof("%s Connected to %v", mc.clientType, master)
 		mc.currentMaster = master
 
 		for {
 			volumeLocation, err := stream.Recv()
 			if err != nil {
-				glog.V(0).Infof("%s failed to receive from %s: %v", mc.name, master, err)
+				glog.V(0).Infof("%s failed to receive from %s: %v", mc.clientType, master, err)
 				return err
 			}
 
@@ -102,18 +102,18 @@ func (mc *MasterClient) tryConnectToMaster(master string) (nextHintedLeader stri
 				PublicUrl: volumeLocation.PublicUrl,
 			}
 			for _, newVid := range volumeLocation.NewVids {
-				glog.V(1).Infof("%s: %s adds volume %d", mc.name, loc.Url, newVid)
+				glog.V(1).Infof("%s: %s adds volume %d", mc.clientType, loc.Url, newVid)
 				mc.addLocation(newVid, loc)
 			}
 			for _, deletedVid := range volumeLocation.DeletedVids {
-				glog.V(1).Infof("%s: %s removes volume %d", mc.name, loc.Url, deletedVid)
+				glog.V(1).Infof("%s: %s removes volume %d", mc.clientType, loc.Url, deletedVid)
 				mc.deleteLocation(deletedVid, loc)
 			}
 		}
 
 	})
 	if gprcErr != nil {
-		glog.V(0).Infof("%s failed to connect with master %v: %v", mc.name, master, gprcErr)
+		glog.V(0).Infof("%s failed to connect with master %v: %v", mc.clientType, master, gprcErr)
 	}
 	return
 }

From 8e23dc078b6790ace902c09ed71963c9ce49acdb Mon Sep 17 00:00:00 2001
From: Chris Lu <chris.lu@gmail.com>
Date: Sun, 12 Apr 2020 20:48:21 -0700
Subject: [PATCH 57/81] refactoring

---
 weed/pb/shared_values.go          | 5 +++++
 weed/server/master_grpc_server.go | 6 +++---
 weed/shell/commands.go            | 2 +-
 3 files changed, 9 insertions(+), 4 deletions(-)
 create mode 100644 weed/pb/shared_values.go

diff --git a/weed/pb/shared_values.go b/weed/pb/shared_values.go
new file mode 100644
index 000000000..cac253ebb
--- /dev/null
+++ b/weed/pb/shared_values.go
@@ -0,0 +1,5 @@
+package pb
+
+const (
+	AdminShellClient = "shell"
+)
\ No newline at end of file
diff --git a/weed/server/master_grpc_server.go b/weed/server/master_grpc_server.go
index 1785a8ccb..a8da9ab2e 100644
--- a/weed/server/master_grpc_server.go
+++ b/weed/server/master_grpc_server.go
@@ -11,6 +11,7 @@ import (
 	"google.golang.org/grpc/peer"
 
 	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/pb"
 	"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
 	"github.com/chrislusf/seaweedfs/weed/storage/backend"
 	"github.com/chrislusf/seaweedfs/weed/storage/needle"
@@ -191,8 +192,8 @@ func (ms *MasterServer) KeepConnected(stream master_pb.Seaweed_KeepConnectedServ
 	peerAddress := findClientAddress(stream.Context(), req.GrpcPort)
 
 	// only one shell can be connected at any time
-	if req.Name == "shell" {
-		if ms.currentAdminShellClient == ""{
+	if req.Name == pb.AdminShellClient {
+		if ms.currentAdminShellClient == "" {
 			ms.currentAdminShellClient = peerAddress
 			defer func() {
 				ms.currentAdminShellClient = ""
@@ -202,7 +203,6 @@ func (ms *MasterServer) KeepConnected(stream master_pb.Seaweed_KeepConnectedServ
 		}
 	}
 
-
 	stopChan := make(chan bool)
 
 	clientName, messageChan := ms.addClient(req.Name, peerAddress)
diff --git a/weed/shell/commands.go b/weed/shell/commands.go
index 41f197328..cbae6c03e 100644
--- a/weed/shell/commands.go
+++ b/weed/shell/commands.go
@@ -43,7 +43,7 @@ var (
 func NewCommandEnv(options ShellOptions) *CommandEnv {
 	return &CommandEnv{
 		env:          make(map[string]string),
-		MasterClient: wdclient.NewMasterClient(options.GrpcDialOption, "shell", 0, strings.Split(*options.Masters, ",")),
+		MasterClient: wdclient.NewMasterClient(options.GrpcDialOption, pb.AdminShellClient, 0, strings.Split(*options.Masters, ",")),
 		option:       options,
 	}
 }

From 7764e0465ce976bb528c27bb9aa25857102570ef Mon Sep 17 00:00:00 2001
From: Chris Lu <chris.lu@gmail.com>
Date: Sun, 12 Apr 2020 21:00:55 -0700
Subject: [PATCH 58/81] refactoring

---
 other/java/client/src/main/proto/filer.proto  |   6 +-
 unmaintained/see_log_entry/see_log_entry.go   |   4 +-
 weed/command/watch.go                         |   2 +-
 weed/filer2/filer_notify.go                   |  10 +-
 weed/filer2/topics.go                         |   4 +-
 weed/filesys/fscache.go                       |   1 -
 weed/pb/filer.proto                           |   6 +-
 weed/pb/filer_pb/filer.pb.go                  | 312 +++++++++---------
 weed/pb/shared_values.go                      |   2 +-
 weed/server/filer_grpc_server_listen.go       |   4 +-
 weed/server/filer_grpc_server_rename.go       |   2 +-
 weed/server/volume_server_handlers_write.go   |   6 +-
 .../backend/memory_map/memory_map_backend.go  |   2 +-
 weed/util/chunk_cache/chunk_cache.go          |   2 +-
 .../chunk_cache/chunk_cache_on_disk_test.go   |   8 +-
 weed/util/log_buffer/log_buffer.go            |   2 +-
 16 files changed, 186 insertions(+), 187 deletions(-)

diff --git a/other/java/client/src/main/proto/filer.proto b/other/java/client/src/main/proto/filer.proto
index 40316f58b..fd2b8ebe3 100644
--- a/other/java/client/src/main/proto/filer.proto
+++ b/other/java/client/src/main/proto/filer.proto
@@ -42,7 +42,7 @@ service SeaweedFiler {
     rpc GetFilerConfiguration (GetFilerConfigurationRequest) returns (GetFilerConfigurationResponse) {
     }
 
-    rpc ListenForEvents (ListenForEventsRequest) returns (stream FullEventNotification) {
+    rpc SubscribeMetadata (SubscribeMetadataRequest) returns (stream SubscribeMetadataResponse) {
     }
 
 }
@@ -234,12 +234,12 @@ message GetFilerConfigurationResponse {
     bool cipher = 7;
 }
 
-message ListenForEventsRequest {
+message SubscribeMetadataRequest {
     string client_name = 1;
     string path_prefix = 2;
     int64 since_ns = 3;
 }
-message FullEventNotification {
+message SubscribeMetadataResponse {
     string directory = 1;
     EventNotification event_notification = 2;
 }
diff --git a/unmaintained/see_log_entry/see_log_entry.go b/unmaintained/see_log_entry/see_log_entry.go
index 69503b65d..34965f6be 100644
--- a/unmaintained/see_log_entry/see_log_entry.go
+++ b/unmaintained/see_log_entry/see_log_entry.go
@@ -61,10 +61,10 @@ func walkLogEntryFile(dst *os.File) error {
 			return nil
 		}
 
-		event := &filer_pb.FullEventNotification{}
+		event := &filer_pb.SubscribeMetadataResponse{}
 		err = proto.Unmarshal(logEntry.Data, event)
 		if err != nil {
-			log.Printf("unexpected unmarshal filer_pb.FullEventNotification: %v", err)
+			log.Printf("unexpected unmarshal filer_pb.SubscribeMetadataResponse: %v", err)
 			return nil
 		}
 
diff --git a/weed/command/watch.go b/weed/command/watch.go
index 2dd6ec211..966040fbb 100644
--- a/weed/command/watch.go
+++ b/weed/command/watch.go
@@ -34,7 +34,7 @@ func runWatch(cmd *Command, args []string) bool {
 
 	watchErr := pb.WithFilerClient(*watchFiler, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
 
-		stream, err := client.ListenForEvents(context.Background(), &filer_pb.ListenForEventsRequest{
+		stream, err := client.SubscribeMetadata(context.Background(), &filer_pb.SubscribeMetadataRequest{
 			ClientName: "watch",
 			PathPrefix: *watchTarget,
 			SinceNs:    0,
diff --git a/weed/filer2/filer_notify.go b/weed/filer2/filer_notify.go
index 18f96658b..c4fcefd54 100644
--- a/weed/filer2/filer_notify.go
+++ b/weed/filer2/filer_notify.go
@@ -53,13 +53,13 @@ func (f *Filer) logMetaEvent(fullpath string, eventNotification *filer_pb.EventN
 
 	dir, _ := util.FullPath(fullpath).DirAndName()
 
-	event := &filer_pb.FullEventNotification{
+	event := &filer_pb.SubscribeMetadataResponse{
 		Directory:         dir,
 		EventNotification: eventNotification,
 	}
 	data, err := proto.Marshal(event)
 	if err != nil {
-		glog.Errorf("failed to marshal filer_pb.FullEventNotification %+v: %v", event, err)
+		glog.Errorf("failed to marshal filer_pb.SubscribeMetadataResponse %+v: %v", event, err)
 		return
 	}
 
@@ -97,11 +97,11 @@ func (f *Filer) ReadLogBuffer(lastReadTime time.Time, eachEventFn func(fullpath
 			return lastReadTime, fmt.Errorf("unexpected unmarshal filer_pb.LogEntry: %v", err)
 		}
 
-		event := &filer_pb.FullEventNotification{}
+		event := &filer_pb.SubscribeMetadataResponse{}
 		err = proto.Unmarshal(logEntry.Data, event)
 		if err != nil {
-			glog.Errorf("unexpected unmarshal filer_pb.FullEventNotification: %v", err)
-			return lastReadTime, fmt.Errorf("unexpected unmarshal filer_pb.FullEventNotification: %v", err)
+			glog.Errorf("unexpected unmarshal filer_pb.SubscribeMetadataResponse: %v", err)
+			return lastReadTime, fmt.Errorf("unexpected unmarshal filer_pb.SubscribeMetadataResponse: %v", err)
 		}
 
 		err = eachEventFn(event.Directory, event.EventNotification)
diff --git a/weed/filer2/topics.go b/weed/filer2/topics.go
index 8ab409774..9c6e5c88d 100644
--- a/weed/filer2/topics.go
+++ b/weed/filer2/topics.go
@@ -1,6 +1,6 @@
 package filer2
 
-const(
-	TopicsDir = "/topics"
+const (
+	TopicsDir    = "/topics"
 	SystemLogDir = TopicsDir + "/.system/log"
 )
diff --git a/weed/filesys/fscache.go b/weed/filesys/fscache.go
index 831f3808c..b146f0615 100644
--- a/weed/filesys/fscache.go
+++ b/weed/filesys/fscache.go
@@ -62,7 +62,6 @@ func (c *FsCache) doSetFsNode(path util.FullPath, node fs.Node) {
 	t.node = node
 }
 
-
 func (c *FsCache) EnsureFsNode(path util.FullPath, genNodeFn func() fs.Node) fs.Node {
 
 	c.Lock()
diff --git a/weed/pb/filer.proto b/weed/pb/filer.proto
index 40316f58b..fd2b8ebe3 100644
--- a/weed/pb/filer.proto
+++ b/weed/pb/filer.proto
@@ -42,7 +42,7 @@ service SeaweedFiler {
     rpc GetFilerConfiguration (GetFilerConfigurationRequest) returns (GetFilerConfigurationResponse) {
     }
 
-    rpc ListenForEvents (ListenForEventsRequest) returns (stream FullEventNotification) {
+    rpc SubscribeMetadata (SubscribeMetadataRequest) returns (stream SubscribeMetadataResponse) {
     }
 
 }
@@ -234,12 +234,12 @@ message GetFilerConfigurationResponse {
     bool cipher = 7;
 }
 
-message ListenForEventsRequest {
+message SubscribeMetadataRequest {
     string client_name = 1;
     string path_prefix = 2;
     int64 since_ns = 3;
 }
-message FullEventNotification {
+message SubscribeMetadataResponse {
     string directory = 1;
     EventNotification event_notification = 2;
 }
diff --git a/weed/pb/filer_pb/filer.pb.go b/weed/pb/filer_pb/filer.pb.go
index e428bb23a..2454395b4 100644
--- a/weed/pb/filer_pb/filer.pb.go
+++ b/weed/pb/filer_pb/filer.pb.go
@@ -39,8 +39,8 @@ It has these top-level messages:
 	StatisticsResponse
 	GetFilerConfigurationRequest
 	GetFilerConfigurationResponse
-	ListenForEventsRequest
-	FullEventNotification
+	SubscribeMetadataRequest
+	SubscribeMetadataResponse
 	LogEntry
 */
 package filer_pb
@@ -1090,56 +1090,56 @@ func (m *GetFilerConfigurationResponse) GetCipher() bool {
 	return false
 }
 
-type ListenForEventsRequest struct {
+type SubscribeMetadataRequest struct {
 	ClientName string `protobuf:"bytes,1,opt,name=client_name,json=clientName" json:"client_name,omitempty"`
 	PathPrefix string `protobuf:"bytes,2,opt,name=path_prefix,json=pathPrefix" json:"path_prefix,omitempty"`
 	SinceNs    int64  `protobuf:"varint,3,opt,name=since_ns,json=sinceNs" json:"since_ns,omitempty"`
 }
 
-func (m *ListenForEventsRequest) Reset()                    { *m = ListenForEventsRequest{} }
-func (m *ListenForEventsRequest) String() string            { return proto.CompactTextString(m) }
-func (*ListenForEventsRequest) ProtoMessage()               {}
-func (*ListenForEventsRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{30} }
+func (m *SubscribeMetadataRequest) Reset()                    { *m = SubscribeMetadataRequest{} }
+func (m *SubscribeMetadataRequest) String() string            { return proto.CompactTextString(m) }
+func (*SubscribeMetadataRequest) ProtoMessage()               {}
+func (*SubscribeMetadataRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{30} }
 
-func (m *ListenForEventsRequest) GetClientName() string {
+func (m *SubscribeMetadataRequest) GetClientName() string {
 	if m != nil {
 		return m.ClientName
 	}
 	return ""
 }
 
-func (m *ListenForEventsRequest) GetPathPrefix() string {
+func (m *SubscribeMetadataRequest) GetPathPrefix() string {
 	if m != nil {
 		return m.PathPrefix
 	}
 	return ""
 }
 
-func (m *ListenForEventsRequest) GetSinceNs() int64 {
+func (m *SubscribeMetadataRequest) GetSinceNs() int64 {
 	if m != nil {
 		return m.SinceNs
 	}
 	return 0
 }
 
-type FullEventNotification struct {
+type SubscribeMetadataResponse struct {
 	Directory         string             `protobuf:"bytes,1,opt,name=directory" json:"directory,omitempty"`
 	EventNotification *EventNotification `protobuf:"bytes,2,opt,name=event_notification,json=eventNotification" json:"event_notification,omitempty"`
 }
 
-func (m *FullEventNotification) Reset()                    { *m = FullEventNotification{} }
-func (m *FullEventNotification) String() string            { return proto.CompactTextString(m) }
-func (*FullEventNotification) ProtoMessage()               {}
-func (*FullEventNotification) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{31} }
+func (m *SubscribeMetadataResponse) Reset()                    { *m = SubscribeMetadataResponse{} }
+func (m *SubscribeMetadataResponse) String() string            { return proto.CompactTextString(m) }
+func (*SubscribeMetadataResponse) ProtoMessage()               {}
+func (*SubscribeMetadataResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{31} }
 
-func (m *FullEventNotification) GetDirectory() string {
+func (m *SubscribeMetadataResponse) GetDirectory() string {
 	if m != nil {
 		return m.Directory
 	}
 	return ""
 }
 
-func (m *FullEventNotification) GetEventNotification() *EventNotification {
+func (m *SubscribeMetadataResponse) GetEventNotification() *EventNotification {
 	if m != nil {
 		return m.EventNotification
 	}
@@ -1209,8 +1209,8 @@ func init() {
 	proto.RegisterType((*StatisticsResponse)(nil), "filer_pb.StatisticsResponse")
 	proto.RegisterType((*GetFilerConfigurationRequest)(nil), "filer_pb.GetFilerConfigurationRequest")
 	proto.RegisterType((*GetFilerConfigurationResponse)(nil), "filer_pb.GetFilerConfigurationResponse")
-	proto.RegisterType((*ListenForEventsRequest)(nil), "filer_pb.ListenForEventsRequest")
-	proto.RegisterType((*FullEventNotification)(nil), "filer_pb.FullEventNotification")
+	proto.RegisterType((*SubscribeMetadataRequest)(nil), "filer_pb.SubscribeMetadataRequest")
+	proto.RegisterType((*SubscribeMetadataResponse)(nil), "filer_pb.SubscribeMetadataResponse")
 	proto.RegisterType((*LogEntry)(nil), "filer_pb.LogEntry")
 }
 
@@ -1236,7 +1236,7 @@ type SeaweedFilerClient interface {
 	DeleteCollection(ctx context.Context, in *DeleteCollectionRequest, opts ...grpc.CallOption) (*DeleteCollectionResponse, error)
 	Statistics(ctx context.Context, in *StatisticsRequest, opts ...grpc.CallOption) (*StatisticsResponse, error)
 	GetFilerConfiguration(ctx context.Context, in *GetFilerConfigurationRequest, opts ...grpc.CallOption) (*GetFilerConfigurationResponse, error)
-	ListenForEvents(ctx context.Context, in *ListenForEventsRequest, opts ...grpc.CallOption) (SeaweedFiler_ListenForEventsClient, error)
+	SubscribeMetadata(ctx context.Context, in *SubscribeMetadataRequest, opts ...grpc.CallOption) (SeaweedFiler_SubscribeMetadataClient, error)
 }
 
 type seaweedFilerClient struct {
@@ -1369,12 +1369,12 @@ func (c *seaweedFilerClient) GetFilerConfiguration(ctx context.Context, in *GetF
 	return out, nil
 }
 
-func (c *seaweedFilerClient) ListenForEvents(ctx context.Context, in *ListenForEventsRequest, opts ...grpc.CallOption) (SeaweedFiler_ListenForEventsClient, error) {
-	stream, err := grpc.NewClientStream(ctx, &_SeaweedFiler_serviceDesc.Streams[1], c.cc, "/filer_pb.SeaweedFiler/ListenForEvents", opts...)
+func (c *seaweedFilerClient) SubscribeMetadata(ctx context.Context, in *SubscribeMetadataRequest, opts ...grpc.CallOption) (SeaweedFiler_SubscribeMetadataClient, error) {
+	stream, err := grpc.NewClientStream(ctx, &_SeaweedFiler_serviceDesc.Streams[1], c.cc, "/filer_pb.SeaweedFiler/SubscribeMetadata", opts...)
 	if err != nil {
 		return nil, err
 	}
-	x := &seaweedFilerListenForEventsClient{stream}
+	x := &seaweedFilerSubscribeMetadataClient{stream}
 	if err := x.ClientStream.SendMsg(in); err != nil {
 		return nil, err
 	}
@@ -1384,17 +1384,17 @@ func (c *seaweedFilerClient) ListenForEvents(ctx context.Context, in *ListenForE
 	return x, nil
 }
 
-type SeaweedFiler_ListenForEventsClient interface {
-	Recv() (*FullEventNotification, error)
+type SeaweedFiler_SubscribeMetadataClient interface {
+	Recv() (*SubscribeMetadataResponse, error)
 	grpc.ClientStream
 }
 
-type seaweedFilerListenForEventsClient struct {
+type seaweedFilerSubscribeMetadataClient struct {
 	grpc.ClientStream
 }
 
-func (x *seaweedFilerListenForEventsClient) Recv() (*FullEventNotification, error) {
-	m := new(FullEventNotification)
+func (x *seaweedFilerSubscribeMetadataClient) Recv() (*SubscribeMetadataResponse, error) {
+	m := new(SubscribeMetadataResponse)
 	if err := x.ClientStream.RecvMsg(m); err != nil {
 		return nil, err
 	}
@@ -1415,7 +1415,7 @@ type SeaweedFilerServer interface {
 	DeleteCollection(context.Context, *DeleteCollectionRequest) (*DeleteCollectionResponse, error)
 	Statistics(context.Context, *StatisticsRequest) (*StatisticsResponse, error)
 	GetFilerConfiguration(context.Context, *GetFilerConfigurationRequest) (*GetFilerConfigurationResponse, error)
-	ListenForEvents(*ListenForEventsRequest, SeaweedFiler_ListenForEventsServer) error
+	SubscribeMetadata(*SubscribeMetadataRequest, SeaweedFiler_SubscribeMetadataServer) error
 }
 
 func RegisterSeaweedFilerServer(s *grpc.Server, srv SeaweedFilerServer) {
@@ -1623,24 +1623,24 @@ func _SeaweedFiler_GetFilerConfiguration_Handler(srv interface{}, ctx context.Co
 	return interceptor(ctx, in, info, handler)
 }
 
-func _SeaweedFiler_ListenForEvents_Handler(srv interface{}, stream grpc.ServerStream) error {
-	m := new(ListenForEventsRequest)
+func _SeaweedFiler_SubscribeMetadata_Handler(srv interface{}, stream grpc.ServerStream) error {
+	m := new(SubscribeMetadataRequest)
 	if err := stream.RecvMsg(m); err != nil {
 		return err
 	}
-	return srv.(SeaweedFilerServer).ListenForEvents(m, &seaweedFilerListenForEventsServer{stream})
+	return srv.(SeaweedFilerServer).SubscribeMetadata(m, &seaweedFilerSubscribeMetadataServer{stream})
 }
 
-type SeaweedFiler_ListenForEventsServer interface {
-	Send(*FullEventNotification) error
+type SeaweedFiler_SubscribeMetadataServer interface {
+	Send(*SubscribeMetadataResponse) error
 	grpc.ServerStream
 }
 
-type seaweedFilerListenForEventsServer struct {
+type seaweedFilerSubscribeMetadataServer struct {
 	grpc.ServerStream
 }
 
-func (x *seaweedFilerListenForEventsServer) Send(m *FullEventNotification) error {
+func (x *seaweedFilerSubscribeMetadataServer) Send(m *SubscribeMetadataResponse) error {
 	return x.ServerStream.SendMsg(m)
 }
 
@@ -1696,8 +1696,8 @@ var _SeaweedFiler_serviceDesc = grpc.ServiceDesc{
 			ServerStreams: true,
 		},
 		{
-			StreamName:    "ListenForEvents",
-			Handler:       _SeaweedFiler_ListenForEvents_Handler,
+			StreamName:    "SubscribeMetadata",
+			Handler:       _SeaweedFiler_SubscribeMetadata_Handler,
 			ServerStreams: true,
 		},
 	},
@@ -1709,122 +1709,122 @@ func init() { proto.RegisterFile("filer.proto", fileDescriptor0) }
 var fileDescriptor0 = []byte{
 	// 1903 bytes of a gzipped FileDescriptorProto
 	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xb4, 0x58, 0x5f, 0x6f, 0xdc, 0xc6,
-	0x11, 0x37, 0xef, 0x74, 0x7f, 0x38, 0x77, 0xe7, 0x48, 0x7b, 0x92, 0x73, 0x3e, 0x4b, 0xb6, 0x42,
-	0xd7, 0xa9, 0x0b, 0x1b, 0xaa, 0xa1, 0xa6, 0x40, 0xd2, 0xb4, 0x0f, 0xb6, 0x2c, 0xa5, 0x6e, 0x6c,
-	0xc5, 0xa0, 0xec, 0x22, 0x45, 0x81, 0x12, 0x14, 0xb9, 0xba, 0xdb, 0x8a, 0x47, 0xb2, 0xbb, 0x4b,
-	0xfd, 0xc9, 0x53, 0xfb, 0x35, 0x0a, 0xf4, 0xa1, 0xdf, 0xa1, 0x8f, 0x45, 0x5f, 0x8a, 0x02, 0x05,
-	0xfa, 0x2d, 0xfa, 0x49, 0x8a, 0x9d, 0x25, 0x79, 0xcb, 0xe3, 0x49, 0x4a, 0x50, 0xe4, 0x6d, 0x77,
-	0x66, 0x76, 0x76, 0x76, 0xfe, 0xfc, 0x66, 0x48, 0xe8, 0x9d, 0xb0, 0x88, 0xf2, 0x9d, 0x94, 0x27,
-	0x32, 0x21, 0x5d, 0xdc, 0x78, 0xe9, 0xb1, 0xf3, 0x15, 0xdc, 0x7b, 0x9d, 0x24, 0xa7, 0x59, 0xfa,
-	0x92, 0x71, 0x1a, 0xc8, 0x84, 0x5f, 0xee, 0xc7, 0x92, 0x5f, 0xba, 0xf4, 0x0f, 0x19, 0x15, 0x92,
-	0x6c, 0x82, 0x1d, 0x16, 0x8c, 0x91, 0xb5, 0x6d, 0x3d, 0xb6, 0xdd, 0x39, 0x81, 0x10, 0x58, 0x89,
-	0xfd, 0x19, 0x1d, 0x35, 0x90, 0x81, 0x6b, 0x67, 0x1f, 0x36, 0x97, 0x2b, 0x14, 0x69, 0x12, 0x0b,
-	0x4a, 0x1e, 0x41, 0x8b, 0x2a, 0x02, 0x6a, 0xeb, 0xed, 0x7e, 0xb0, 0x53, 0x98, 0xb2, 0xa3, 0xe5,
-	0x34, 0xd7, 0xf9, 0x87, 0x05, 0xe4, 0x35, 0x13, 0x52, 0x11, 0x19, 0x15, 0xdf, 0xce, 0x9e, 0x3b,
-	0xd0, 0x4e, 0x39, 0x3d, 0x61, 0x17, 0xb9, 0x45, 0xf9, 0x8e, 0x3c, 0x85, 0x35, 0x21, 0x7d, 0x2e,
-	0x0f, 0x78, 0x32, 0x3b, 0x60, 0x11, 0x3d, 0x54, 0x46, 0x37, 0x51, 0xa4, 0xce, 0x20, 0x3b, 0x40,
-	0x58, 0x1c, 0x44, 0x99, 0x60, 0x67, 0xf4, 0xa8, 0xe0, 0x8e, 0x56, 0xb6, 0xad, 0xc7, 0x5d, 0x77,
-	0x09, 0x87, 0xac, 0x43, 0x2b, 0x62, 0x33, 0x26, 0x47, 0xad, 0x6d, 0xeb, 0xf1, 0xc0, 0xd5, 0x1b,
-	0xe7, 0xe7, 0x30, 0xac, 0xd8, 0xff, 0xdd, 0x9e, 0xff, 0x97, 0x06, 0xb4, 0x90, 0x50, 0xfa, 0xd8,
-	0x9a, 0xfb, 0x98, 0x7c, 0x04, 0x7d, 0x26, 0xbc, 0xb9, 0x23, 0x1a, 0x68, 0x5b, 0x8f, 0x89, 0xd2,
-	0xe7, 0xe4, 0x09, 0xb4, 0x83, 0x69, 0x16, 0x9f, 0x8a, 0x51, 0x73, 0xbb, 0xf9, 0xb8, 0xb7, 0x3b,
-	0x9c, 0x5f, 0xa4, 0x1e, 0xba, 0xa7, 0x78, 0x6e, 0x2e, 0x42, 0x3e, 0x05, 0xf0, 0xa5, 0xe4, 0xec,
-	0x38, 0x93, 0x54, 0xe0, 0x4b, 0x7b, 0xbb, 0x23, 0xe3, 0x40, 0x26, 0xe8, 0xf3, 0x92, 0xef, 0x1a,
-	0xb2, 0xe4, 0x33, 0xe8, 0xd2, 0x0b, 0x49, 0xe3, 0x90, 0x86, 0xa3, 0x16, 0x5e, 0xb4, 0xb5, 0xf0,
-	0xa2, 0x9d, 0xfd, 0x9c, 0xaf, 0xdf, 0x57, 0x8a, 0x8f, 0x3f, 0x87, 0x41, 0x85, 0x45, 0x56, 0xa1,
-	0x79, 0x4a, 0x8b, 0xa8, 0xaa, 0xa5, 0xf2, 0xec, 0x99, 0x1f, 0x65, 0x3a, 0xc1, 0xfa, 0xae, 0xde,
-	0xfc, 0xac, 0xf1, 0xa9, 0xe5, 0xbc, 0x04, 0xfb, 0x20, 0x8b, 0xa2, 0xf2, 0x60, 0xc8, 0x78, 0x71,
-	0x30, 0x64, 0x7c, 0xee, 0xe5, 0xc6, 0xb5, 0x5e, 0xfe, 0xbb, 0x05, 0x6b, 0xfb, 0x67, 0x34, 0x96,
-	0x87, 0x89, 0x64, 0x27, 0x2c, 0xf0, 0x25, 0x4b, 0x62, 0xf2, 0x14, 0xec, 0x24, 0x0a, 0xbd, 0x6b,
-	0xc3, 0xd4, 0x4d, 0xa2, 0xdc, 0xea, 0xa7, 0x60, 0xc7, 0xf4, 0xdc, 0xbb, 0xf6, 0xba, 0x6e, 0x4c,
-	0xcf, 0xb5, 0xf4, 0x43, 0x18, 0x84, 0x34, 0xa2, 0x92, 0x7a, 0x65, 0x74, 0x54, 0xe8, 0xfa, 0x9a,
-	0xb8, 0xa7, 0xc3, 0xf1, 0x31, 0x7c, 0xa0, 0x54, 0xa6, 0x3e, 0xa7, 0xb1, 0xf4, 0x52, 0x5f, 0x4e,
-	0x31, 0x26, 0xb6, 0x3b, 0x88, 0xe9, 0xf9, 0x5b, 0xa4, 0xbe, 0xf5, 0xe5, 0xd4, 0xf9, 0x5b, 0x03,
-	0xec, 0x32, 0x98, 0xe4, 0x43, 0xe8, 0xa8, 0x6b, 0x3d, 0x16, 0xe6, 0x9e, 0x68, 0xab, 0xed, 0xab,
-	0x50, 0x55, 0x45, 0x72, 0x72, 0x22, 0xa8, 0x44, 0xf3, 0x9a, 0x6e, 0xbe, 0x53, 0x99, 0x25, 0xd8,
-	0x37, 0xba, 0x10, 0x56, 0x5c, 0x5c, 0x2b, 0x8f, 0xcf, 0x24, 0x9b, 0x51, 0xbc, 0xb0, 0xe9, 0xea,
-	0x0d, 0x19, 0x42, 0x8b, 0x7a, 0xd2, 0x9f, 0x60, 0x86, 0xdb, 0xee, 0x0a, 0x7d, 0xe7, 0x4f, 0xc8,
-	0x0f, 0xe0, 0xb6, 0x48, 0x32, 0x1e, 0x50, 0xaf, 0xb8, 0xb6, 0x8d, 0xdc, 0xbe, 0xa6, 0x1e, 0xe8,
-	0xcb, 0x1d, 0x68, 0x9e, 0xb0, 0x70, 0xd4, 0x41, 0xc7, 0xac, 0x56, 0x93, 0xf0, 0x55, 0xe8, 0x2a,
-	0x26, 0xf9, 0x31, 0x40, 0xa9, 0x29, 0x1c, 0x75, 0xaf, 0x10, 0xb5, 0x0b, 0xbd, 0x21, 0xd9, 0x02,
-	0x08, 0x58, 0x3a, 0xa5, 0xdc, 0x53, 0x09, 0x63, 0x63, 0x72, 0xd8, 0x9a, 0xf2, 0x25, 0xbd, 0x54,
-	0x6c, 0x26, 0xbc, 0xc9, 0x37, 0x2c, 0x4d, 0x69, 0x38, 0x02, 0xf4, 0xb0, 0xcd, 0xc4, 0x17, 0x9a,
-	0xe0, 0x7c, 0x0d, 0xed, 0xdc, 0xb8, 0x7b, 0x60, 0x9f, 0x25, 0x51, 0x36, 0x2b, 0x9d, 0x36, 0x70,
-	0xbb, 0x9a, 0xf0, 0x2a, 0x24, 0x77, 0x01, 0x51, 0x12, 0xaf, 0x68, 0xa0, 0x8b, 0xd0, 0xbf, 0xea,
-	0x82, 0x3b, 0xd0, 0x0e, 0x92, 0xe4, 0x94, 0x69, 0xdf, 0x75, 0xdc, 0x7c, 0xe7, 0xfc, 0xb1, 0x09,
-	0xb7, 0xab, 0xc5, 0xa2, 0xae, 0x40, 0x2d, 0xe8, 0x69, 0x0b, 0xd5, 0xa0, 0xda, 0xa3, 0x8a, 0xb7,
-	0x1b, 0xa6, 0xb7, 0x8b, 0x23, 0xb3, 0x24, 0xd4, 0x17, 0x0c, 0xf4, 0x91, 0x37, 0x49, 0x48, 0x55,
-	0xae, 0x67, 0x2c, 0xc4, 0xf0, 0x0c, 0x5c, 0xb5, 0x54, 0x94, 0x09, 0x0b, 0x73, 0xf0, 0x51, 0x4b,
-	0x34, 0x8f, 0xa3, 0xde, 0xb6, 0x0e, 0xb8, 0xde, 0xa9, 0x80, 0xcf, 0x14, 0xb5, 0xa3, 0xa3, 0xa8,
-	0xd6, 0x64, 0x1b, 0x7a, 0x9c, 0xa6, 0x51, 0x9e, 0xfb, 0xe8, 0x7c, 0xdb, 0x35, 0x49, 0xe4, 0x3e,
-	0x40, 0x90, 0x44, 0x11, 0x0d, 0x50, 0xc0, 0x46, 0x01, 0x83, 0xa2, 0xf2, 0x4e, 0xca, 0xc8, 0x13,
-	0x34, 0x40, 0x57, 0xb7, 0xdc, 0xb6, 0x94, 0xd1, 0x11, 0x0d, 0xd4, 0x3b, 0x32, 0x41, 0xb9, 0x87,
-	0xf0, 0xd5, 0xc3, 0x73, 0x5d, 0x45, 0x40, 0x90, 0xdd, 0x02, 0x98, 0xf0, 0x24, 0x4b, 0x35, 0xb7,
-	0xbf, 0xdd, 0x54, 0x48, 0x8e, 0x14, 0x64, 0x3f, 0x82, 0xdb, 0xe2, 0x72, 0x16, 0xb1, 0xf8, 0xd4,
-	0x93, 0x3e, 0x9f, 0x50, 0x39, 0x1a, 0xe8, 0x0a, 0xc8, 0xa9, 0xef, 0x90, 0xa8, 0xde, 0x3e, 0x0b,
-	0x7f, 0x3a, 0xba, 0x8d, 0x19, 0xa0, 0x96, 0x4e, 0x0a, 0x64, 0x8f, 0x53, 0x5f, 0xd2, 0xef, 0xd0,
-	0xc6, 0xbe, 0x1d, 0x5a, 0x90, 0x0d, 0x68, 0x27, 0x1e, 0xbd, 0x08, 0xa2, 0xbc, 0x68, 0x5b, 0xc9,
-	0xfe, 0x45, 0x10, 0x39, 0x4f, 0x60, 0x58, 0xb9, 0x31, 0x07, 0xfa, 0x75, 0x68, 0x51, 0xce, 0x93,
-	0x02, 0x96, 0xf4, 0xc6, 0xf9, 0x0d, 0x90, 0xf7, 0x69, 0xf8, 0x7d, 0x98, 0xe7, 0x6c, 0xc0, 0xb0,
-	0xa2, 0x5a, 0xdb, 0xe1, 0xfc, 0xcb, 0x02, 0xf2, 0x12, 0xd1, 0xe5, 0xff, 0x6b, 0xec, 0xaa, 0xde,
-	0x55, 0xd3, 0xd1, 0xe8, 0x15, 0xfa, 0xd2, 0xcf, 0x5b, 0x62, 0x9f, 0x09, 0xad, 0xff, 0xa5, 0x2f,
-	0xfd, 0xbc, 0x35, 0x71, 0x1a, 0x64, 0x5c, 0x75, 0x49, 0x4c, 0x4b, 0x6c, 0x4d, 0x6e, 0x41, 0x22,
-	0x9f, 0xc0, 0x1d, 0x36, 0x89, 0x13, 0x4e, 0xe7, 0x62, 0x9e, 0x76, 0x55, 0x1b, 0x85, 0xd7, 0x35,
-	0xb7, 0x3c, 0xb0, 0x8f, 0x9e, 0x7b, 0x02, 0xc3, 0xca, 0x33, 0xae, 0x75, 0xf3, 0x9f, 0x2d, 0x18,
-	0x3d, 0x97, 0xc9, 0x8c, 0x05, 0x2e, 0x55, 0xc6, 0x57, 0x9e, 0xfe, 0x10, 0x06, 0x0a, 0xdf, 0x17,
-	0x9f, 0xdf, 0x4f, 0xa2, 0x70, 0xde, 0x3f, 0xef, 0x82, 0x82, 0x78, 0xcf, 0xf0, 0x42, 0x27, 0x89,
-	0x42, 0xcc, 0xcd, 0x87, 0xa0, 0x70, 0xd8, 0x38, 0xaf, 0x27, 0x89, 0x7e, 0x4c, 0xcf, 0x2b, 0xe7,
-	0x95, 0x10, 0x9e, 0xd7, 0xe0, 0xdd, 0x89, 0xe9, 0xb9, 0x3a, 0xef, 0xdc, 0x83, 0xbb, 0x4b, 0x6c,
-	0xcb, 0xc3, 0xf5, 0x6f, 0x0b, 0x86, 0xcf, 0x85, 0x60, 0x93, 0xf8, 0xd7, 0x08, 0x44, 0x85, 0xd1,
-	0xeb, 0xd0, 0x0a, 0x92, 0x2c, 0x96, 0x68, 0x6c, 0xcb, 0xd5, 0x9b, 0x85, 0xda, 0x6c, 0xd4, 0x6a,
-	0x73, 0xa1, 0xba, 0x9b, 0xf5, 0xea, 0x36, 0xaa, 0x77, 0xa5, 0x52, 0xbd, 0x0f, 0xa0, 0xa7, 0x82,
-	0xec, 0x05, 0x34, 0x96, 0x94, 0xe7, 0xc8, 0x0f, 0x8a, 0xb4, 0x87, 0x14, 0x25, 0x60, 0x76, 0x28,
-	0x0d, 0xfe, 0x90, 0xce, 0xdb, 0xd3, 0x7f, 0x2d, 0x58, 0xaf, 0x3e, 0x25, 0x8f, 0xd9, 0x95, 0x9d,
-	0x4a, 0x81, 0x1b, 0x8f, 0xf2, 0x77, 0xa8, 0xa5, 0x82, 0x89, 0x34, 0x3b, 0x8e, 0x58, 0xe0, 0x29,
-	0x86, 0xb6, 0xdf, 0xd6, 0x94, 0xf7, 0x3c, 0x9a, 0x7b, 0x65, 0xc5, 0xf4, 0x0a, 0x81, 0x15, 0x3f,
-	0x93, 0xd3, 0xa2, 0x5b, 0xa9, 0xf5, 0x82, 0xa7, 0xda, 0x37, 0x79, 0xaa, 0x53, 0xf7, 0x54, 0x99,
-	0x69, 0x5d, 0x33, 0xd3, 0x3e, 0x81, 0xa1, 0x1e, 0x77, 0xab, 0xe1, 0xda, 0x02, 0x28, 0x3b, 0x8b,
-	0x18, 0x59, 0x1a, 0xde, 0x8a, 0xd6, 0x22, 0x9c, 0x5f, 0x80, 0xfd, 0x3a, 0xd1, 0x7a, 0x05, 0x79,
-	0x06, 0x76, 0x54, 0x6c, 0x50, 0xb4, 0xb7, 0x4b, 0xe6, 0x35, 0x5e, 0xc8, 0xb9, 0x73, 0x21, 0xe7,
-	0x73, 0xe8, 0x16, 0xe4, 0xc2, 0x67, 0xd6, 0x55, 0x3e, 0x6b, 0x2c, 0xf8, 0xcc, 0xf9, 0xa7, 0x05,
-	0xeb, 0x55, 0x93, 0xf3, 0xb0, 0xbc, 0x87, 0x41, 0x79, 0x85, 0x37, 0xf3, 0xd3, 0xdc, 0x96, 0x67,
-	0xa6, 0x2d, 0xf5, 0x63, 0xa5, 0x81, 0xe2, 0x8d, 0x9f, 0xea, 0x5c, 0xee, 0x47, 0x06, 0x69, 0xfc,
-	0x0e, 0xd6, 0x6a, 0x22, 0x4b, 0x66, 0xbd, 0x1f, 0x99, 0xb3, 0x5e, 0x65, 0x5e, 0x2d, 0x4f, 0x9b,
-	0x03, 0xe0, 0x67, 0xf0, 0xa1, 0x86, 0x83, 0xbd, 0x32, 0x86, 0x85, 0xef, 0xab, 0xa1, 0xb6, 0x16,
-	0x43, 0xed, 0x8c, 0x61, 0x54, 0x3f, 0x9a, 0x97, 0xdf, 0x04, 0xd6, 0x8e, 0xa4, 0x2f, 0x99, 0x90,
-	0x2c, 0x28, 0x3f, 0x3a, 0x16, 0x72, 0xc3, 0xba, 0xa9, 0x47, 0xd6, 0xeb, 0x70, 0x15, 0x9a, 0x52,
-	0x16, 0xf9, 0xab, 0x96, 0x2a, 0x0a, 0xc4, 0xbc, 0x29, 0x8f, 0xc1, 0xf7, 0x70, 0x95, 0xca, 0x07,
-	0x99, 0x48, 0x3f, 0xd2, 0x33, 0xc8, 0x0a, 0xce, 0x20, 0x36, 0x52, 0x70, 0x08, 0xd1, 0x6d, 0x3a,
-	0xd4, 0xdc, 0x96, 0x9e, 0x50, 0x14, 0x01, 0x99, 0x5b, 0x00, 0x58, 0xaa, 0xba, 0xca, 0xda, 0xfa,
-	0xac, 0xa2, 0xec, 0x29, 0x82, 0x73, 0x1f, 0x36, 0xbf, 0xa0, 0x52, 0x4d, 0x53, 0x7c, 0x2f, 0x89,
-	0x4f, 0xd8, 0x24, 0xe3, 0xbe, 0x11, 0x0a, 0xe7, 0x3f, 0x16, 0x6c, 0x5d, 0x21, 0x90, 0x3f, 0x78,
-	0x04, 0x9d, 0x99, 0x2f, 0x24, 0xe5, 0x45, 0x95, 0x14, 0xdb, 0x45, 0x57, 0x34, 0x6e, 0x72, 0x45,
-	0xb3, 0xe6, 0x8a, 0x0d, 0x68, 0xcf, 0xfc, 0x0b, 0x6f, 0x76, 0x9c, 0x8f, 0x4b, 0xad, 0x99, 0x7f,
-	0xf1, 0xe6, 0x18, 0x91, 0x8d, 0x71, 0xef, 0x38, 0x0b, 0x4e, 0xa9, 0x14, 0x25, 0xb2, 0x31, 0xfe,
-	0x42, 0x53, 0x70, 0x7e, 0xc2, 0x61, 0x12, 0x61, 0xa0, 0xeb, 0xe6, 0x3b, 0x27, 0x83, 0x3b, 0xea,
-	0x93, 0x8e, 0xc6, 0x07, 0x09, 0xc7, 0xcf, 0x86, 0x32, 0x43, 0x1e, 0x40, 0x2f, 0x88, 0x98, 0xc2,
-	0x42, 0xe3, 0x5b, 0x0d, 0x34, 0x09, 0x7b, 0x06, 0x82, 0xa5, 0x9c, 0x7a, 0x95, 0xcf, 0x53, 0x50,
-	0xa4, 0xb7, 0xfa, 0x13, 0xf5, 0x2e, 0x74, 0x05, 0x8b, 0x03, 0xea, 0xc5, 0xfa, 0x9b, 0xa0, 0xe9,
-	0x76, 0x70, 0x7f, 0x28, 0x9c, 0x3f, 0x59, 0xb0, 0x81, 0x1f, 0x3b, 0xb5, 0x2f, 0x95, 0xeb, 0x9b,
-	0xf8, 0xaf, 0x80, 0xd0, 0x33, 0xb4, 0xc9, 0x38, 0x93, 0x97, 0xd7, 0x3d, 0x63, 0x88, 0x58, 0x54,
-	0xeb, 0xae, 0xd1, 0x45, 0x92, 0xe3, 0x2b, 0xc4, 0x99, 0xe8, 0xda, 0x1d, 0x42, 0x4b, 0x0a, 0x0f,
-	0xb1, 0x4a, 0xd9, 0xb9, 0x22, 0xc5, 0xa1, 0x20, 0x4f, 0x81, 0xa4, 0x3e, 0x97, 0x4c, 0x49, 0xab,
-	0x91, 0xd9, 0x9b, 0xfa, 0x62, 0x8a, 0x97, 0xb5, 0xdc, 0xd5, 0x92, 0xf3, 0x25, 0xbd, 0xfc, 0xa5,
-	0x2f, 0xa6, 0x0a, 0xa1, 0x71, 0x82, 0x68, 0xe2, 0xe0, 0x86, 0xeb, 0xdd, 0xbf, 0x76, 0xa1, 0x7f,
-	0x44, 0xfd, 0x73, 0x4a, 0x43, 0xcc, 0x17, 0x32, 0x29, 0x70, 0xaa, 0xfa, 0x27, 0x81, 0x3c, 0x5a,
-	0x04, 0xa4, 0xa5, 0xbf, 0x2e, 0xc6, 0x1f, 0xdf, 0x24, 0x96, 0x97, 0xfc, 0x2d, 0x72, 0x08, 0x3d,
-	0xe3, 0x53, 0x9d, 0x6c, 0x1a, 0x07, 0x6b, 0x7f, 0x20, 0xc6, 0x5b, 0x57, 0x70, 0x0b, 0x6d, 0xcf,
-	0x2c, 0xf2, 0x1a, 0x7a, 0xc6, 0x44, 0x68, 0xea, 0xab, 0x8f, 0xa6, 0xa6, 0xbe, 0x25, 0x63, 0xa4,
-	0x73, 0x4b, 0x69, 0x33, 0xe6, 0x3a, 0x53, 0x5b, 0x7d, 0x92, 0x34, 0xb5, 0x2d, 0x1b, 0x06, 0x51,
-	0x9b, 0x31, 0x46, 0x99, 0xda, 0xea, 0x43, 0xa2, 0xa9, 0x6d, 0xc9, 0xec, 0xe5, 0xdc, 0x22, 0xbf,
-	0x83, 0xb5, 0xda, 0x28, 0x43, 0x9c, 0xf9, 0xa9, 0xab, 0x66, 0xb0, 0xf1, 0xc3, 0x6b, 0x65, 0x4a,
-	0xfd, 0x5f, 0x41, 0xdf, 0x9c, 0x20, 0x88, 0x61, 0xd0, 0x92, 0x21, 0x69, 0x7c, 0xff, 0x2a, 0xb6,
-	0xa9, 0xd0, 0x6c, 0x62, 0xa6, 0xc2, 0x25, 0x6d, 0xdc, 0x54, 0xb8, 0xac, 0xf7, 0x39, 0xb7, 0xc8,
-	0x6f, 0x61, 0x75, 0xb1, 0x99, 0x90, 0x8f, 0x16, 0xdd, 0x56, 0xeb, 0x51, 0x63, 0xe7, 0x3a, 0x91,
-	0x52, 0xf9, 0x2b, 0x80, 0x79, 0x8f, 0x20, 0x46, 0xcd, 0xd6, 0x7a, 0xd4, 0x78, 0x73, 0x39, 0xb3,
-	0x54, 0xf5, 0x7b, 0xd8, 0x58, 0x0a, 0xc4, 0xc4, 0x28, 0x93, 0xeb, 0xa0, 0x7c, 0xfc, 0xc3, 0x1b,
-	0xe5, 0xca, 0xbb, 0xbe, 0x86, 0x0f, 0x16, 0x70, 0x92, 0x6c, 0x57, 0xab, 0xa6, 0x0e, 0xa1, 0xe3,
-	0x07, 0xe6, 0xff, 0xa6, 0x25, 0x60, 0xa7, 0x2a, 0xeb, 0xc5, 0x7d, 0x58, 0x15, 0x1a, 0x22, 0x4e,
-	0xc4, 0x8e, 0x86, 0xd7, 0x17, 0x80, 0xb6, 0xbc, 0xe5, 0x89, 0x4c, 0x8e, 0xdb, 0xf8, 0x7b, 0xf3,
-	0x27, 0xff, 0x0b, 0x00, 0x00, 0xff, 0xff, 0x0d, 0xae, 0xfb, 0x80, 0xed, 0x14, 0x00, 0x00,
+	0x11, 0x37, 0xef, 0x3f, 0xe7, 0xee, 0x1c, 0x69, 0x4f, 0x4e, 0x4e, 0x67, 0xc9, 0x51, 0xe8, 0x3a,
+	0x75, 0x61, 0x43, 0x35, 0xd4, 0x14, 0x48, 0x9a, 0xf6, 0xc1, 0x96, 0xe5, 0xd4, 0x8d, 0xad, 0x18,
+	0x94, 0x5d, 0xb4, 0x28, 0x50, 0x96, 0x47, 0xae, 0xee, 0xb6, 0xe2, 0x91, 0xec, 0xee, 0x52, 0x7f,
+	0xf2, 0xd4, 0x97, 0x7e, 0x89, 0x02, 0xfd, 0x00, 0x7d, 0xef, 0x63, 0xd1, 0x97, 0xa2, 0x40, 0x81,
+	0x7e, 0x8b, 0x7e, 0x92, 0x62, 0x67, 0x49, 0xde, 0xf2, 0xfe, 0x48, 0x09, 0x8a, 0xbc, 0xed, 0xce,
+	0xcc, 0xce, 0xce, 0xce, 0x9f, 0xdf, 0x0c, 0x09, 0xdd, 0x53, 0x16, 0x51, 0xbe, 0x9f, 0xf2, 0x44,
+	0x26, 0xa4, 0x83, 0x1b, 0x2f, 0x1d, 0x3b, 0x5f, 0xc1, 0xdd, 0x57, 0x49, 0x72, 0x96, 0xa5, 0xcf,
+	0x19, 0xa7, 0x81, 0x4c, 0xf8, 0xd5, 0x51, 0x2c, 0xf9, 0x95, 0x4b, 0xff, 0x90, 0x51, 0x21, 0xc9,
+	0x0e, 0xd8, 0x61, 0xc1, 0x18, 0x5a, 0x7b, 0xd6, 0x43, 0xdb, 0x9d, 0x13, 0x08, 0x81, 0x46, 0xec,
+	0xcf, 0xe8, 0xb0, 0x86, 0x0c, 0x5c, 0x3b, 0x47, 0xb0, 0xb3, 0x5a, 0xa1, 0x48, 0x93, 0x58, 0x50,
+	0xf2, 0x00, 0x9a, 0x54, 0x11, 0x50, 0x5b, 0xf7, 0xe0, 0xbd, 0xfd, 0xc2, 0x94, 0x7d, 0x2d, 0xa7,
+	0xb9, 0xce, 0x3f, 0x2c, 0x20, 0xaf, 0x98, 0x90, 0x8a, 0xc8, 0xa8, 0xf8, 0x66, 0xf6, 0xbc, 0x0f,
+	0xad, 0x94, 0xd3, 0x53, 0x76, 0x99, 0x5b, 0x94, 0xef, 0xc8, 0x63, 0xd8, 0x14, 0xd2, 0xe7, 0xf2,
+	0x05, 0x4f, 0x66, 0x2f, 0x58, 0x44, 0x8f, 0x95, 0xd1, 0x75, 0x14, 0x59, 0x66, 0x90, 0x7d, 0x20,
+	0x2c, 0x0e, 0xa2, 0x4c, 0xb0, 0x73, 0x7a, 0x52, 0x70, 0x87, 0x8d, 0x3d, 0xeb, 0x61, 0xc7, 0x5d,
+	0xc1, 0x21, 0x5b, 0xd0, 0x8c, 0xd8, 0x8c, 0xc9, 0x61, 0x73, 0xcf, 0x7a, 0xd8, 0x77, 0xf5, 0xc6,
+	0xf9, 0x29, 0x0c, 0x2a, 0xf6, 0x7f, 0xbb, 0xe7, 0xff, 0xa5, 0x06, 0x4d, 0x24, 0x94, 0x3e, 0xb6,
+	0xe6, 0x3e, 0x26, 0x1f, 0x41, 0x8f, 0x09, 0x6f, 0xee, 0x88, 0x1a, 0xda, 0xd6, 0x65, 0xa2, 0xf4,
+	0x39, 0x79, 0x04, 0xad, 0x60, 0x9a, 0xc5, 0x67, 0x62, 0x58, 0xdf, 0xab, 0x3f, 0xec, 0x1e, 0x0c,
+	0xe6, 0x17, 0xa9, 0x87, 0x1e, 0x2a, 0x9e, 0x9b, 0x8b, 0x90, 0x4f, 0x01, 0x7c, 0x29, 0x39, 0x1b,
+	0x67, 0x92, 0x0a, 0x7c, 0x69, 0xf7, 0x60, 0x68, 0x1c, 0xc8, 0x04, 0x7d, 0x5a, 0xf2, 0x5d, 0x43,
+	0x96, 0x7c, 0x06, 0x1d, 0x7a, 0x29, 0x69, 0x1c, 0xd2, 0x70, 0xd8, 0xc4, 0x8b, 0x76, 0x17, 0x5e,
+	0xb4, 0x7f, 0x94, 0xf3, 0xf5, 0xfb, 0x4a, 0xf1, 0xd1, 0xe7, 0xd0, 0xaf, 0xb0, 0xc8, 0x06, 0xd4,
+	0xcf, 0x68, 0x11, 0x55, 0xb5, 0x54, 0x9e, 0x3d, 0xf7, 0xa3, 0x4c, 0x27, 0x58, 0xcf, 0xd5, 0x9b,
+	0x9f, 0xd4, 0x3e, 0xb5, 0x9c, 0xe7, 0x60, 0xbf, 0xc8, 0xa2, 0xa8, 0x3c, 0x18, 0x32, 0x5e, 0x1c,
+	0x0c, 0x19, 0x9f, 0x7b, 0xb9, 0x76, 0xad, 0x97, 0xff, 0x6e, 0xc1, 0xe6, 0xd1, 0x39, 0x8d, 0xe5,
+	0x71, 0x22, 0xd9, 0x29, 0x0b, 0x7c, 0xc9, 0x92, 0x98, 0x3c, 0x06, 0x3b, 0x89, 0x42, 0xef, 0xda,
+	0x30, 0x75, 0x92, 0x28, 0xb7, 0xfa, 0x31, 0xd8, 0x31, 0xbd, 0xf0, 0xae, 0xbd, 0xae, 0x13, 0xd3,
+	0x0b, 0x2d, 0x7d, 0x1f, 0xfa, 0x21, 0x8d, 0xa8, 0xa4, 0x5e, 0x19, 0x1d, 0x15, 0xba, 0x9e, 0x26,
+	0x1e, 0xea, 0x70, 0x7c, 0x0c, 0xef, 0x29, 0x95, 0xa9, 0xcf, 0x69, 0x2c, 0xbd, 0xd4, 0x97, 0x53,
+	0x8c, 0x89, 0xed, 0xf6, 0x63, 0x7a, 0xf1, 0x06, 0xa9, 0x6f, 0x7c, 0x39, 0x75, 0xfe, 0x56, 0x03,
+	0xbb, 0x0c, 0x26, 0xf9, 0x00, 0xda, 0xea, 0x5a, 0x8f, 0x85, 0xb9, 0x27, 0x5a, 0x6a, 0xfb, 0x32,
+	0x54, 0x55, 0x91, 0x9c, 0x9e, 0x0a, 0x2a, 0xd1, 0xbc, 0xba, 0x9b, 0xef, 0x54, 0x66, 0x09, 0xf6,
+	0xb5, 0x2e, 0x84, 0x86, 0x8b, 0x6b, 0xe5, 0xf1, 0x99, 0x64, 0x33, 0x8a, 0x17, 0xd6, 0x5d, 0xbd,
+	0x21, 0x03, 0x68, 0x52, 0x4f, 0xfa, 0x13, 0xcc, 0x70, 0xdb, 0x6d, 0xd0, 0xb7, 0xfe, 0x84, 0x7c,
+	0x0f, 0x6e, 0x8b, 0x24, 0xe3, 0x01, 0xf5, 0x8a, 0x6b, 0x5b, 0xc8, 0xed, 0x69, 0xea, 0x0b, 0x7d,
+	0xb9, 0x03, 0xf5, 0x53, 0x16, 0x0e, 0xdb, 0xe8, 0x98, 0x8d, 0x6a, 0x12, 0xbe, 0x0c, 0x5d, 0xc5,
+	0x24, 0x3f, 0x04, 0x28, 0x35, 0x85, 0xc3, 0xce, 0x1a, 0x51, 0xbb, 0xd0, 0x1b, 0x92, 0x5d, 0x80,
+	0x80, 0xa5, 0x53, 0xca, 0x3d, 0x95, 0x30, 0x36, 0x26, 0x87, 0xad, 0x29, 0x5f, 0xd2, 0x2b, 0xc5,
+	0x66, 0xc2, 0x9b, 0x7c, 0xcd, 0xd2, 0x94, 0x86, 0x43, 0x40, 0x0f, 0xdb, 0x4c, 0x7c, 0xa1, 0x09,
+	0xce, 0xaf, 0xa0, 0x95, 0x1b, 0x77, 0x17, 0xec, 0xf3, 0x24, 0xca, 0x66, 0xa5, 0xd3, 0xfa, 0x6e,
+	0x47, 0x13, 0x5e, 0x86, 0x64, 0x1b, 0x10, 0x25, 0xf1, 0x8a, 0x1a, 0xba, 0x08, 0xfd, 0xab, 0x2e,
+	0x78, 0x1f, 0x5a, 0x41, 0x92, 0x9c, 0x31, 0xed, 0xbb, 0xb6, 0x9b, 0xef, 0x9c, 0x3f, 0xd6, 0xe1,
+	0x76, 0xb5, 0x58, 0xd4, 0x15, 0xa8, 0x05, 0x3d, 0x6d, 0xa1, 0x1a, 0x54, 0x7b, 0x52, 0xf1, 0x76,
+	0xcd, 0xf4, 0x76, 0x71, 0x64, 0x96, 0x84, 0xfa, 0x82, 0xbe, 0x3e, 0xf2, 0x3a, 0x09, 0xa9, 0xca,
+	0xf5, 0x8c, 0x85, 0x18, 0x9e, 0xbe, 0xab, 0x96, 0x8a, 0x32, 0x61, 0x61, 0x0e, 0x3e, 0x6a, 0x89,
+	0xe6, 0x71, 0xd4, 0xdb, 0xd2, 0x01, 0xd7, 0x3b, 0x15, 0xf0, 0x99, 0xa2, 0xb6, 0x75, 0x14, 0xd5,
+	0x9a, 0xec, 0x41, 0x97, 0xd3, 0x34, 0xca, 0x73, 0x1f, 0x9d, 0x6f, 0xbb, 0x26, 0x89, 0xdc, 0x03,
+	0x08, 0x92, 0x28, 0xa2, 0x01, 0x0a, 0xd8, 0x28, 0x60, 0x50, 0x54, 0xde, 0x49, 0x19, 0x79, 0x82,
+	0x06, 0xe8, 0xea, 0xa6, 0xdb, 0x92, 0x32, 0x3a, 0xa1, 0x81, 0x7a, 0x47, 0x26, 0x28, 0xf7, 0x10,
+	0xbe, 0xba, 0x78, 0xae, 0xa3, 0x08, 0x08, 0xb2, 0xbb, 0x00, 0x13, 0x9e, 0x64, 0xa9, 0xe6, 0xf6,
+	0xf6, 0xea, 0x0a, 0xc9, 0x91, 0x82, 0xec, 0x07, 0x70, 0x5b, 0x5c, 0xcd, 0x22, 0x16, 0x9f, 0x79,
+	0xd2, 0xe7, 0x13, 0x2a, 0x87, 0x7d, 0x5d, 0x01, 0x39, 0xf5, 0x2d, 0x12, 0xd5, 0xdb, 0x67, 0xe1,
+	0x8f, 0x87, 0xb7, 0x31, 0x03, 0xd4, 0xd2, 0x49, 0x81, 0x1c, 0x72, 0xea, 0x4b, 0xfa, 0x2d, 0xda,
+	0xd8, 0x37, 0x43, 0x0b, 0x72, 0x07, 0x5a, 0x89, 0x47, 0x2f, 0x83, 0x28, 0x2f, 0xda, 0x66, 0x72,
+	0x74, 0x19, 0x44, 0xce, 0x23, 0x18, 0x54, 0x6e, 0xcc, 0x81, 0x7e, 0x0b, 0x9a, 0x94, 0xf3, 0xa4,
+	0x80, 0x25, 0xbd, 0x71, 0x7e, 0x0d, 0xe4, 0x5d, 0x1a, 0x7e, 0x17, 0xe6, 0x39, 0x77, 0x60, 0x50,
+	0x51, 0xad, 0xed, 0x70, 0xfe, 0x65, 0x01, 0x79, 0x8e, 0xe8, 0xf2, 0xff, 0x35, 0x76, 0x55, 0xef,
+	0xaa, 0xe9, 0x68, 0xf4, 0x0a, 0x7d, 0xe9, 0xe7, 0x2d, 0xb1, 0xc7, 0x84, 0xd6, 0xff, 0xdc, 0x97,
+	0x7e, 0xde, 0x9a, 0x38, 0x0d, 0x32, 0xae, 0xba, 0x24, 0xa6, 0x25, 0xb6, 0x26, 0xb7, 0x20, 0x91,
+	0x4f, 0xe0, 0x7d, 0x36, 0x89, 0x13, 0x4e, 0xe7, 0x62, 0x9e, 0x76, 0x55, 0x0b, 0x85, 0xb7, 0x34,
+	0xb7, 0x3c, 0x70, 0x84, 0x9e, 0x7b, 0x04, 0x83, 0xca, 0x33, 0xae, 0x75, 0xf3, 0x9f, 0x2d, 0x18,
+	0x3e, 0x95, 0xc9, 0x8c, 0x05, 0x2e, 0x55, 0xc6, 0x57, 0x9e, 0x7e, 0x1f, 0xfa, 0x0a, 0xdf, 0x17,
+	0x9f, 0xdf, 0x4b, 0xa2, 0x70, 0xde, 0x3f, 0xb7, 0x41, 0x41, 0xbc, 0x67, 0x78, 0xa1, 0x9d, 0x44,
+	0x21, 0xe6, 0xe6, 0x7d, 0x50, 0x38, 0x6c, 0x9c, 0xd7, 0x93, 0x44, 0x2f, 0xa6, 0x17, 0x95, 0xf3,
+	0x4a, 0x08, 0xcf, 0x6b, 0xf0, 0x6e, 0xc7, 0xf4, 0x42, 0x9d, 0x77, 0xee, 0xc2, 0xf6, 0x0a, 0xdb,
+	0xf2, 0x70, 0xfd, 0xdb, 0x82, 0xc1, 0x53, 0x21, 0xd8, 0x24, 0xfe, 0x25, 0x02, 0x51, 0x61, 0xf4,
+	0x16, 0x34, 0x83, 0x24, 0x8b, 0x25, 0x1a, 0xdb, 0x74, 0xf5, 0x66, 0xa1, 0x36, 0x6b, 0x4b, 0xb5,
+	0xb9, 0x50, 0xdd, 0xf5, 0xe5, 0xea, 0x36, 0xaa, 0xb7, 0x51, 0xa9, 0xde, 0x0f, 0xa1, 0xab, 0x82,
+	0xec, 0x05, 0x34, 0x96, 0x94, 0xe7, 0xc8, 0x0f, 0x8a, 0x74, 0x88, 0x14, 0x25, 0x60, 0x76, 0x28,
+	0x0d, 0xfe, 0x90, 0xce, 0xdb, 0xd3, 0x7f, 0x2d, 0xd8, 0xaa, 0x3e, 0x25, 0x8f, 0xd9, 0xda, 0x4e,
+	0xa5, 0xc0, 0x8d, 0x47, 0xf9, 0x3b, 0xd4, 0x52, 0xc1, 0x44, 0x9a, 0x8d, 0x23, 0x16, 0x78, 0x8a,
+	0xa1, 0xed, 0xb7, 0x35, 0xe5, 0x1d, 0x8f, 0xe6, 0x5e, 0x69, 0x98, 0x5e, 0x21, 0xd0, 0xf0, 0x33,
+	0x39, 0x2d, 0xba, 0x95, 0x5a, 0x2f, 0x78, 0xaa, 0x75, 0x93, 0xa7, 0xda, 0xcb, 0x9e, 0x2a, 0x33,
+	0xad, 0x63, 0x66, 0xda, 0x27, 0x30, 0xd0, 0xe3, 0x6e, 0x35, 0x5c, 0xbb, 0x00, 0x65, 0x67, 0x11,
+	0x43, 0x4b, 0xc3, 0x5b, 0xd1, 0x5a, 0x84, 0xf3, 0x33, 0xb0, 0x5f, 0x25, 0x5a, 0xaf, 0x20, 0x4f,
+	0xc0, 0x8e, 0x8a, 0x0d, 0x8a, 0x76, 0x0f, 0xc8, 0xbc, 0xc6, 0x0b, 0x39, 0x77, 0x2e, 0xe4, 0x7c,
+	0x0e, 0x9d, 0x82, 0x5c, 0xf8, 0xcc, 0x5a, 0xe7, 0xb3, 0xda, 0x82, 0xcf, 0x9c, 0x7f, 0x5a, 0xb0,
+	0x55, 0x35, 0x39, 0x0f, 0xcb, 0x3b, 0xe8, 0x97, 0x57, 0x78, 0x33, 0x3f, 0xcd, 0x6d, 0x79, 0x62,
+	0xda, 0xb2, 0x7c, 0xac, 0x34, 0x50, 0xbc, 0xf6, 0x53, 0x9d, 0xcb, 0xbd, 0xc8, 0x20, 0x8d, 0xde,
+	0xc2, 0xe6, 0x92, 0xc8, 0x8a, 0x59, 0xef, 0x07, 0xe6, 0xac, 0x57, 0x99, 0x57, 0xcb, 0xd3, 0xe6,
+	0x00, 0xf8, 0x19, 0x7c, 0xa0, 0xe1, 0xe0, 0xb0, 0x8c, 0x61, 0xe1, 0xfb, 0x6a, 0xa8, 0xad, 0xc5,
+	0x50, 0x3b, 0x23, 0x18, 0x2e, 0x1f, 0xcd, 0xcb, 0x6f, 0x02, 0x9b, 0x27, 0xd2, 0x97, 0x4c, 0x48,
+	0x16, 0x94, 0x1f, 0x1d, 0x0b, 0xb9, 0x61, 0xdd, 0xd4, 0x23, 0x97, 0xeb, 0x70, 0x03, 0xea, 0x52,
+	0x16, 0xf9, 0xab, 0x96, 0x2a, 0x0a, 0xc4, 0xbc, 0x29, 0x8f, 0xc1, 0x77, 0x70, 0x95, 0xca, 0x07,
+	0x99, 0x48, 0x3f, 0xd2, 0x33, 0x48, 0x03, 0x67, 0x10, 0x1b, 0x29, 0x38, 0x84, 0xe8, 0x36, 0x1d,
+	0x6a, 0x6e, 0x53, 0x4f, 0x28, 0x8a, 0x80, 0xcc, 0x5d, 0x00, 0x2c, 0x55, 0x5d, 0x65, 0x2d, 0x7d,
+	0x56, 0x51, 0x0e, 0x15, 0xc1, 0xb9, 0x07, 0x3b, 0x5f, 0x50, 0xa9, 0xa6, 0x29, 0x7e, 0x98, 0xc4,
+	0xa7, 0x6c, 0x92, 0x71, 0xdf, 0x08, 0x85, 0xf3, 0x1f, 0x0b, 0x76, 0xd7, 0x08, 0xe4, 0x0f, 0x1e,
+	0x42, 0x7b, 0xe6, 0x0b, 0x49, 0x79, 0x51, 0x25, 0xc5, 0x76, 0xd1, 0x15, 0xb5, 0x9b, 0x5c, 0x51,
+	0x5f, 0x72, 0xc5, 0x1d, 0x68, 0xcd, 0xfc, 0x4b, 0x6f, 0x36, 0xce, 0xc7, 0xa5, 0xe6, 0xcc, 0xbf,
+	0x7c, 0x3d, 0x46, 0x64, 0x63, 0xdc, 0x1b, 0x67, 0xc1, 0x19, 0x95, 0xa2, 0x44, 0x36, 0xc6, 0x9f,
+	0x69, 0x0a, 0xce, 0x4f, 0x38, 0x4c, 0x22, 0x0c, 0x74, 0xdc, 0x7c, 0xe7, 0x5c, 0xc0, 0xf0, 0x24,
+	0x1b, 0x8b, 0x80, 0xb3, 0x31, 0x7d, 0x4d, 0xa5, 0xaf, 0xc0, 0xb0, 0xc8, 0x91, 0x0f, 0xa1, 0x1b,
+	0x44, 0x4c, 0xa1, 0xa1, 0xf1, 0xb5, 0x06, 0x9a, 0x84, 0x5d, 0x03, 0xe1, 0x52, 0x4e, 0xbd, 0xca,
+	0x07, 0x2a, 0x28, 0xd2, 0x1b, 0xfd, 0x91, 0xba, 0x0d, 0x1d, 0xc1, 0xe2, 0x80, 0x7a, 0xb1, 0xfe,
+	0x2a, 0xa8, 0xbb, 0x6d, 0xdc, 0x1f, 0x0b, 0xe7, 0x4f, 0x16, 0x6c, 0xaf, 0xb8, 0x39, 0x77, 0xe1,
+	0xf5, 0xad, 0xfc, 0x17, 0x40, 0xe8, 0x39, 0xda, 0x65, 0x7c, 0xe3, 0xe4, 0x45, 0x76, 0xd7, 0x18,
+	0x25, 0x16, 0x3f, 0x83, 0xdc, 0x4d, 0xba, 0x48, 0x72, 0x7c, 0x85, 0x3b, 0x13, 0x5d, 0xc1, 0x03,
+	0x68, 0x4a, 0xe1, 0x21, 0x62, 0x29, 0x5b, 0x1b, 0x52, 0x1c, 0x0b, 0xf2, 0x18, 0x48, 0xea, 0x73,
+	0xc9, 0x94, 0xb4, 0x1a, 0x9c, 0xbd, 0xa9, 0x2f, 0xa6, 0x78, 0x59, 0xd3, 0xdd, 0x28, 0x39, 0x5f,
+	0xd2, 0xab, 0x9f, 0xfb, 0x62, 0xaa, 0x70, 0x1a, 0xe7, 0x88, 0x3a, 0x8e, 0x6f, 0xb8, 0x3e, 0xf8,
+	0x6b, 0x07, 0x7a, 0x27, 0xd4, 0xbf, 0xa0, 0x34, 0xc4, 0xac, 0x21, 0x93, 0x02, 0xad, 0xaa, 0xff,
+	0x13, 0xc8, 0x83, 0x45, 0x58, 0x5a, 0xf9, 0x03, 0x63, 0xf4, 0xf1, 0x4d, 0x62, 0x79, 0xe1, 0xdf,
+	0x22, 0xc7, 0xd0, 0x35, 0x3e, 0xd8, 0xc9, 0x8e, 0x71, 0x70, 0xe9, 0x3f, 0xc4, 0x68, 0x77, 0x0d,
+	0xb7, 0xd0, 0xf6, 0xc4, 0x22, 0xaf, 0xa0, 0x6b, 0xcc, 0x85, 0xa6, 0xbe, 0xe5, 0x01, 0xd5, 0xd4,
+	0xb7, 0x62, 0x98, 0x74, 0x6e, 0x29, 0x6d, 0xc6, 0x74, 0x67, 0x6a, 0x5b, 0x9e, 0x27, 0x4d, 0x6d,
+	0xab, 0x46, 0x42, 0xd4, 0x66, 0x0c, 0x53, 0xa6, 0xb6, 0xe5, 0x51, 0xd1, 0xd4, 0xb6, 0x62, 0x02,
+	0x73, 0x6e, 0x91, 0xdf, 0xc2, 0xe6, 0xd2, 0x40, 0x43, 0x9c, 0xf9, 0xa9, 0x75, 0x93, 0xd8, 0xe8,
+	0xfe, 0xb5, 0x32, 0xa5, 0xfe, 0xaf, 0xa0, 0x67, 0xce, 0x11, 0xc4, 0x30, 0x68, 0xc5, 0xa8, 0x34,
+	0xba, 0xb7, 0x8e, 0x6d, 0x2a, 0x34, 0x5b, 0x99, 0xa9, 0x70, 0x45, 0x33, 0x37, 0x15, 0xae, 0xea,
+	0x80, 0xce, 0x2d, 0xf2, 0x1b, 0xd8, 0x58, 0x6c, 0x29, 0xe4, 0xa3, 0x45, 0xb7, 0x2d, 0x75, 0xaa,
+	0x91, 0x73, 0x9d, 0x48, 0xa9, 0xfc, 0x25, 0xc0, 0xbc, 0x53, 0x10, 0xa3, 0x66, 0x97, 0x3a, 0xd5,
+	0x68, 0x67, 0x35, 0xb3, 0x54, 0xf5, 0x7b, 0xb8, 0xb3, 0x12, 0x8e, 0x89, 0x51, 0x26, 0xd7, 0x01,
+	0xfa, 0xe8, 0xfb, 0x37, 0xca, 0x95, 0x77, 0xfd, 0x0e, 0x36, 0x97, 0x30, 0xcb, 0xcc, 0x8a, 0x75,
+	0x50, 0x6a, 0x66, 0xc5, 0x5a, 0xd0, 0x53, 0x15, 0xf6, 0xec, 0x1e, 0x6c, 0x08, 0x0d, 0x15, 0xa7,
+	0x62, 0x5f, 0x43, 0xed, 0x33, 0x40, 0x9b, 0xde, 0xf0, 0x44, 0x26, 0xe3, 0x16, 0xfe, 0xec, 0xfc,
+	0xd1, 0xff, 0x02, 0x00, 0x00, 0xff, 0xff, 0x37, 0x58, 0xa5, 0x1f, 0xfb, 0x14, 0x00, 0x00,
 }
diff --git a/weed/pb/shared_values.go b/weed/pb/shared_values.go
index cac253ebb..acc3bb56d 100644
--- a/weed/pb/shared_values.go
+++ b/weed/pb/shared_values.go
@@ -2,4 +2,4 @@ package pb
 
 const (
 	AdminShellClient = "shell"
-)
\ No newline at end of file
+)
diff --git a/weed/server/filer_grpc_server_listen.go b/weed/server/filer_grpc_server_listen.go
index e2d932a99..76bf834ea 100644
--- a/weed/server/filer_grpc_server_listen.go
+++ b/weed/server/filer_grpc_server_listen.go
@@ -10,7 +10,7 @@ import (
 	"github.com/chrislusf/seaweedfs/weed/util"
 )
 
-func (fs *FilerServer) ListenForEvents(req *filer_pb.ListenForEventsRequest, stream filer_pb.SeaweedFiler_ListenForEventsServer) error {
+func (fs *FilerServer) SubscribeMetadata(req *filer_pb.SubscribeMetadataRequest, stream filer_pb.SeaweedFiler_SubscribeMetadataServer) error {
 
 	peerAddress := findClientAddress(stream.Context(), 0)
 
@@ -46,7 +46,7 @@ func (fs *FilerServer) ListenForEvents(req *filer_pb.ListenForEventsRequest, str
 				return nil
 			}
 
-			message := &filer_pb.FullEventNotification{
+			message := &filer_pb.SubscribeMetadataResponse{
 				Directory:         dirPath,
 				EventNotification: eventNotification,
 			}
diff --git a/weed/server/filer_grpc_server_rename.go b/weed/server/filer_grpc_server_rename.go
index c18587b45..71e2c7d17 100644
--- a/weed/server/filer_grpc_server_rename.go
+++ b/weed/server/filer_grpc_server_rename.go
@@ -117,7 +117,7 @@ func (fs *FilerServer) moveSelfEntry(ctx context.Context, oldParent util.FullPat
 
 	events.newEntries = append(events.newEntries, newEntry)
 
-	if moveFolderSubEntries!=nil {
+	if moveFolderSubEntries != nil {
 		if moveChildrenErr := moveFolderSubEntries(); moveChildrenErr != nil {
 			return moveChildrenErr
 		}
diff --git a/weed/server/volume_server_handlers_write.go b/weed/server/volume_server_handlers_write.go
index 968ece11f..227cb0b28 100644
--- a/weed/server/volume_server_handlers_write.go
+++ b/weed/server/volume_server_handlers_write.go
@@ -167,10 +167,10 @@ func setEtag(w http.ResponseWriter, etag string) {
 	}
 }
 
-func getEtag(resp *http.Response) (etag string){
+func getEtag(resp *http.Response) (etag string) {
 	etag = resp.Header.Get("ETag")
-	if strings.HasPrefix(etag, "\"") && strings.HasSuffix(etag, "\""){
-		return etag[1:len(etag)-1]
+	if strings.HasPrefix(etag, "\"") && strings.HasSuffix(etag, "\"") {
+		return etag[1 : len(etag)-1]
 	}
 	return
 }
diff --git a/weed/storage/backend/memory_map/memory_map_backend.go b/weed/storage/backend/memory_map/memory_map_backend.go
index 7fc2b4920..8ff03d9af 100644
--- a/weed/storage/backend/memory_map/memory_map_backend.go
+++ b/weed/storage/backend/memory_map/memory_map_backend.go
@@ -6,7 +6,7 @@ import (
 )
 
 var (
-	// _ backend.BackendStorageFile = &MemoryMappedFile{} // remove this to break import cycle
+// _ backend.BackendStorageFile = &MemoryMappedFile{} // remove this to break import cycle
 )
 
 type MemoryMappedFile struct {
diff --git a/weed/util/chunk_cache/chunk_cache.go b/weed/util/chunk_cache/chunk_cache.go
index 9f2a0518f..7c4a77304 100644
--- a/weed/util/chunk_cache/chunk_cache.go
+++ b/weed/util/chunk_cache/chunk_cache.go
@@ -131,4 +131,4 @@ func (c *ChunkCache) Shutdown() {
 	for _, diskCache := range c.diskCaches {
 		diskCache.Shutdown()
 	}
-}
\ No newline at end of file
+}
diff --git a/weed/util/chunk_cache/chunk_cache_on_disk_test.go b/weed/util/chunk_cache/chunk_cache_on_disk_test.go
index 256b10139..f93daf5a7 100644
--- a/weed/util/chunk_cache/chunk_cache_on_disk_test.go
+++ b/weed/util/chunk_cache/chunk_cache_on_disk_test.go
@@ -21,11 +21,11 @@ func TestOnDisk(t *testing.T) {
 
 	writeCount := 5
 	type test_data struct {
-		data []byte
+		data   []byte
 		fileId string
 	}
 	testData := make([]*test_data, writeCount)
-	for i:=0;i<writeCount;i++{
+	for i := 0; i < writeCount; i++ {
 		buff := make([]byte, 1024*1024)
 		rand.Read(buff)
 		testData[i] = &test_data{
@@ -35,7 +35,7 @@ func TestOnDisk(t *testing.T) {
 		cache.SetChunk(testData[i].fileId, testData[i].data)
 	}
 
-	for i:=0;i<writeCount;i++{
+	for i := 0; i < writeCount; i++ {
 		data := cache.GetChunk(testData[i].fileId)
 		if bytes.Compare(data, testData[i].data) != 0 {
 			t.Errorf("failed to write to and read from cache: %d", i)
@@ -46,7 +46,7 @@ func TestOnDisk(t *testing.T) {
 
 	cache = NewChunkCache(0, tmpDir, totalDiskSizeMb, segmentCount)
 
-	for i:=0;i<writeCount;i++{
+	for i := 0; i < writeCount; i++ {
 		data := cache.GetChunk(testData[i].fileId)
 		if bytes.Compare(data, testData[i].data) != 0 {
 			t.Errorf("failed to write to and read from cache: %d", i)
diff --git a/weed/util/log_buffer/log_buffer.go b/weed/util/log_buffer/log_buffer.go
index 3b3396128..04fabdf8c 100644
--- a/weed/util/log_buffer/log_buffer.go
+++ b/weed/util/log_buffer/log_buffer.go
@@ -163,7 +163,7 @@ func (m *LogBuffer) ReadFromBuffer(lastReadTime time.Time) (ts time.Time, buffer
 	/*
 		for i, pos := range m.idx {
 			logEntry, ts := readTs(m.buf, pos)
-			event := &filer_pb.FullEventNotification{}
+			event := &filer_pb.SubscribeMetadataResponse{}
 			proto.Unmarshal(logEntry.Data, event)
 			entry := event.EventNotification.OldEntry
 			if entry == nil {

From 5c9e91393f358dad74b96ad4e0cdbb7fa0f63752 Mon Sep 17 00:00:00 2001
From: Chris Lu <chris.lu@gmail.com>
Date: Mon, 13 Apr 2020 10:25:48 -0700
Subject: [PATCH 59/81] 1.74

---
 k8s/seaweedfs/Chart.yaml  | 2 +-
 k8s/seaweedfs/values.yaml | 2 +-
 weed/util/constants.go    | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/k8s/seaweedfs/Chart.yaml b/k8s/seaweedfs/Chart.yaml
index 30a3ca83e..921a60cfa 100644
--- a/k8s/seaweedfs/Chart.yaml
+++ b/k8s/seaweedfs/Chart.yaml
@@ -1,4 +1,4 @@
 apiVersion: v1
 description: SeaweedFS
 name: seaweedfs
-version: 1.73
+version: 1.74
diff --git a/k8s/seaweedfs/values.yaml b/k8s/seaweedfs/values.yaml
index c46fd2a3f..68df44237 100644
--- a/k8s/seaweedfs/values.yaml
+++ b/k8s/seaweedfs/values.yaml
@@ -4,7 +4,7 @@ global:
   registry: ""
   repository: ""
   imageName: chrislusf/seaweedfs
-  imageTag: "1.73"
+  imageTag: "1.74"
   imagePullPolicy: IfNotPresent
   imagePullSecrets: imagepullsecret
   restartPolicy: Always
diff --git a/weed/util/constants.go b/weed/util/constants.go
index 0fd43e318..e5b512524 100644
--- a/weed/util/constants.go
+++ b/weed/util/constants.go
@@ -5,5 +5,5 @@ import (
 )
 
 var (
-	VERSION = fmt.Sprintf("%s %d.%d", sizeLimit, 1, 73)
+	VERSION = fmt.Sprintf("%s %d.%d", sizeLimit, 1, 74)
 )

From d8f5985e5e2a51719b7bf40a99c2728b475dbbf4 Mon Sep 17 00:00:00 2001
From: Chris Lu <chris.lu@gmail.com>
Date: Mon, 13 Apr 2020 12:58:45 -0700
Subject: [PATCH 60/81] add logs

---
 weed/command/master.go | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/weed/command/master.go b/weed/command/master.go
index 1be60426f..6171f8e83 100644
--- a/weed/command/master.go
+++ b/weed/command/master.go
@@ -47,7 +47,7 @@ func init() {
 	m.ip = cmdMaster.Flag.String("ip", "localhost", "master <ip>|<server> address")
 	m.ipBind = cmdMaster.Flag.String("ip.bind", "0.0.0.0", "ip address to bind to")
 	m.metaFolder = cmdMaster.Flag.String("mdir", os.TempDir(), "data directory to store meta data")
-	m.peers = cmdMaster.Flag.String("peers", "", "all master nodes in comma separated ip:port list, example: 127.0.0.1:9093,127.0.0.1:9094")
+	m.peers = cmdMaster.Flag.String("peers", "", "all master nodes in comma separated ip:port list, example: 127.0.0.1:9093,127.0.0.1:9094,127.0.0.1:9095")
 	m.volumeSizeLimitMB = cmdMaster.Flag.Uint("volumeSizeLimitMB", 30*1000, "Master stops directing writes to oversized volumes.")
 	m.volumePreallocate = cmdMaster.Flag.Bool("volumePreallocate", false, "Preallocate disk space for volumes.")
 	m.pulseSeconds = cmdMaster.Flag.Int("pulseSeconds", 5, "number of seconds between heartbeats")
@@ -147,6 +147,7 @@ func startMaster(masterOption MasterOptions, masterWhiteList []string) {
 }
 
 func checkPeers(masterIp string, masterPort int, peers string) (masterAddress string, cleanedPeers []string) {
+	glog.V(0).Infof("current: %s:%d peers:%s", masterIp, masterPort, peers)
 	masterAddress = masterIp + ":" + strconv.Itoa(masterPort)
 	if peers != "" {
 		cleanedPeers = strings.Split(peers, ",")

From b556d3d03595620364781fcc10b5bc509112c400 Mon Sep 17 00:00:00 2001
From: wuyuxiang <wuyuxiang@corp.netease.com>
Date: Tue, 14 Apr 2020 10:22:40 +0800
Subject: [PATCH 61/81] Fix data race about config

---
 weed/util/config.go | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/weed/util/config.go b/weed/util/config.go
index 33809d44d..7b6e92f08 100644
--- a/weed/util/config.go
+++ b/weed/util/config.go
@@ -42,7 +42,8 @@ func LoadConfiguration(configFileName string, required bool) (loaded bool) {
 }
 
 func GetViper() *viper.Viper {
-	v := viper.GetViper()
+	v := &viper.Viper{}
+	*v = *viper.GetViper()
 	v.AutomaticEnv()
 	v.SetEnvPrefix("weed")
 	v.SetEnvKeyReplacer(strings.NewReplacer(".", "_"))

From f282ed444baf6676c22df1b7c35964dd73d2c04a Mon Sep 17 00:00:00 2001
From: Chris Lu <chris.lu@gmail.com>
Date: Mon, 13 Apr 2020 21:58:10 -0700
Subject: [PATCH 62/81] refactoring

---
 weed/filer2/filechunks.go                     | 21 +++--
 weed/filer2/reader_at.go                      |  2 +-
 weed/filer2/stream.go                         |  4 +-
 weed/replication/sink/azuresink/azure_sink.go |  2 +-
 weed/replication/sink/b2sink/b2_sink.go       |  2 +-
 weed/replication/sink/gcssink/gcs_sink.go     |  2 +-
 weed/util/chunk_cache/chunk_cache.go          | 86 ++++++-------------
 .../chunk_cache/chunk_cache_on_disk_test.go   |  6 +-
 weed/util/chunk_cache/on_disk_cache_layer.go  | 83 ++++++++++++++++++
 9 files changed, 130 insertions(+), 78 deletions(-)
 create mode 100644 weed/util/chunk_cache/on_disk_cache_layer.go

diff --git a/weed/filer2/filechunks.go b/weed/filer2/filechunks.go
index 48eaeea27..2ddfb3c30 100644
--- a/weed/filer2/filechunks.go
+++ b/weed/filer2/filechunks.go
@@ -85,11 +85,15 @@ type ChunkView struct {
 	Offset      int64
 	Size        uint64
 	LogicOffset int64
-	IsFullChunk bool
+	ChunkSize   uint64
 	CipherKey   []byte
 	IsGzipped   bool
 }
 
+func (cv *ChunkView) IsFullChunk() bool {
+	return cv.Size == cv.ChunkSize
+}
+
 func ViewFromChunks(chunks []*filer_pb.FileChunk, offset int64, size int64) (views []*ChunkView) {
 
 	visibles := NonOverlappingVisibleIntervals(chunks)
@@ -111,13 +115,12 @@ func ViewFromVisibleIntervals(visibles []VisibleInterval, offset int64, size int
 	for _, chunk := range visibles {
 
 		if chunk.start <= offset && offset < chunk.stop && offset < stop {
-			isFullChunk := chunk.isFullChunk && chunk.start == offset && chunk.stop <= stop
 			views = append(views, &ChunkView{
 				FileId:      chunk.fileId,
 				Offset:      offset - chunk.start, // offset is the data starting location in this file id
 				Size:        uint64(min(chunk.stop, stop) - offset),
 				LogicOffset: offset,
-				IsFullChunk: isFullChunk,
+				ChunkSize:   chunk.chunkSize,
 				CipherKey:   chunk.cipherKey,
 				IsGzipped:   chunk.isGzipped,
 			})
@@ -146,7 +149,7 @@ var bufPool = sync.Pool{
 
 func MergeIntoVisibles(visibles, newVisibles []VisibleInterval, chunk *filer_pb.FileChunk) []VisibleInterval {
 
-	newV := newVisibleInterval(chunk.Offset, chunk.Offset+int64(chunk.Size), chunk.GetFileIdString(), chunk.Mtime, true, chunk.CipherKey, chunk.IsGzipped)
+	newV := newVisibleInterval(chunk.Offset, chunk.Offset+int64(chunk.Size), chunk.GetFileIdString(), chunk.Mtime, chunk.Size, chunk.CipherKey, chunk.IsGzipped)
 
 	length := len(visibles)
 	if length == 0 {
@@ -160,11 +163,11 @@ func MergeIntoVisibles(visibles, newVisibles []VisibleInterval, chunk *filer_pb.
 	logPrintf("  before", visibles)
 	for _, v := range visibles {
 		if v.start < chunk.Offset && chunk.Offset < v.stop {
-			newVisibles = append(newVisibles, newVisibleInterval(v.start, chunk.Offset, v.fileId, v.modifiedTime, false, v.cipherKey, v.isGzipped))
+			newVisibles = append(newVisibles, newVisibleInterval(v.start, chunk.Offset, v.fileId, v.modifiedTime, chunk.Size, v.cipherKey, v.isGzipped))
 		}
 		chunkStop := chunk.Offset + int64(chunk.Size)
 		if v.start < chunkStop && chunkStop < v.stop {
-			newVisibles = append(newVisibles, newVisibleInterval(chunkStop, v.stop, v.fileId, v.modifiedTime, false, v.cipherKey, v.isGzipped))
+			newVisibles = append(newVisibles, newVisibleInterval(chunkStop, v.stop, v.fileId, v.modifiedTime, chunk.Size, v.cipherKey, v.isGzipped))
 		}
 		if chunkStop <= v.start || v.stop <= chunk.Offset {
 			newVisibles = append(newVisibles, v)
@@ -216,18 +219,18 @@ type VisibleInterval struct {
 	stop         int64
 	modifiedTime int64
 	fileId       string
-	isFullChunk  bool
+	chunkSize    uint64
 	cipherKey    []byte
 	isGzipped    bool
 }
 
-func newVisibleInterval(start, stop int64, fileId string, modifiedTime int64, isFullChunk bool, cipherKey []byte, isGzipped bool) VisibleInterval {
+func newVisibleInterval(start, stop int64, fileId string, modifiedTime int64, chunkSize uint64, cipherKey []byte, isGzipped bool) VisibleInterval {
 	return VisibleInterval{
 		start:        start,
 		stop:         stop,
 		fileId:       fileId,
 		modifiedTime: modifiedTime,
-		isFullChunk:  isFullChunk,
+		chunkSize:    chunkSize,
 		cipherKey:    cipherKey,
 		isGzipped:    isGzipped,
 	}
diff --git a/weed/filer2/reader_at.go b/weed/filer2/reader_at.go
index c1ad1677f..f56ef6388 100644
--- a/weed/filer2/reader_at.go
+++ b/weed/filer2/reader_at.go
@@ -106,7 +106,7 @@ func (c *ChunkReadAt) fetchChunkData(chunkView *ChunkView) (data []byte, err err
 	// fmt.Printf("fetching %s [%d,%d)\n", chunkView.FileId, chunkView.LogicOffset, chunkView.LogicOffset+int64(chunkView.Size))
 
 	hasDataInCache := false
-	chunkData := c.chunkCache.GetChunk(chunkView.FileId)
+	chunkData := c.chunkCache.GetChunk(chunkView.FileId, chunkView.ChunkSize)
 	if chunkData != nil {
 		glog.V(3).Infof("cache hit %s [%d,%d)", chunkView.FileId, chunkView.LogicOffset, chunkView.LogicOffset+int64(chunkView.Size))
 		hasDataInCache = true
diff --git a/weed/filer2/stream.go b/weed/filer2/stream.go
index bf3781ae2..3cb69f72b 100644
--- a/weed/filer2/stream.go
+++ b/weed/filer2/stream.go
@@ -31,7 +31,7 @@ func StreamContent(masterClient *wdclient.MasterClient, w io.Writer, chunks []*f
 	for _, chunkView := range chunkViews {
 
 		urlString := fileId2Url[chunkView.FileId]
-		err := util.ReadUrlAsStream(urlString, chunkView.CipherKey, chunkView.IsGzipped, chunkView.IsFullChunk, chunkView.Offset, int(chunkView.Size), func(data []byte) {
+		err := util.ReadUrlAsStream(urlString, chunkView.CipherKey, chunkView.IsGzipped, chunkView.IsFullChunk(), chunkView.Offset, int(chunkView.Size), func(data []byte) {
 			w.Write(data)
 		})
 		if err != nil {
@@ -128,7 +128,7 @@ func (c *ChunkStreamReader) fetchChunkToBuffer(chunkView *ChunkView) error {
 		return err
 	}
 	var buffer bytes.Buffer
-	err = util.ReadUrlAsStream(urlString, chunkView.CipherKey, chunkView.IsGzipped, chunkView.IsFullChunk, chunkView.Offset, int(chunkView.Size), func(data []byte) {
+	err = util.ReadUrlAsStream(urlString, chunkView.CipherKey, chunkView.IsGzipped, chunkView.IsFullChunk(), chunkView.Offset, int(chunkView.Size), func(data []byte) {
 		buffer.Write(data)
 	})
 	if err != nil {
diff --git a/weed/replication/sink/azuresink/azure_sink.go b/weed/replication/sink/azuresink/azure_sink.go
index d75dbe9af..aef97c06e 100644
--- a/weed/replication/sink/azuresink/azure_sink.go
+++ b/weed/replication/sink/azuresink/azure_sink.go
@@ -115,7 +115,7 @@ func (g *AzureSink) CreateEntry(key string, entry *filer_pb.Entry) error {
 		}
 
 		var writeErr error
-		readErr := util.ReadUrlAsStream(fileUrl, nil, false, chunk.IsFullChunk, chunk.Offset, int(chunk.Size), func(data []byte) {
+		readErr := util.ReadUrlAsStream(fileUrl, nil, false, chunk.IsFullChunk(), chunk.Offset, int(chunk.Size), func(data []byte) {
 			_, writeErr = appendBlobURL.AppendBlock(context.Background(), bytes.NewReader(data), azblob.AppendBlobAccessConditions{}, nil)
 		})
 
diff --git a/weed/replication/sink/b2sink/b2_sink.go b/weed/replication/sink/b2sink/b2_sink.go
index b5d410a75..1e7d82ed4 100644
--- a/weed/replication/sink/b2sink/b2_sink.go
+++ b/weed/replication/sink/b2sink/b2_sink.go
@@ -103,7 +103,7 @@ func (g *B2Sink) CreateEntry(key string, entry *filer_pb.Entry) error {
 		}
 
 		var writeErr error
-		readErr := util.ReadUrlAsStream(fileUrl, nil, false, chunk.IsFullChunk, chunk.Offset, int(chunk.Size), func(data []byte) {
+		readErr := util.ReadUrlAsStream(fileUrl, nil, false, chunk.IsFullChunk(), chunk.Offset, int(chunk.Size), func(data []byte) {
 			_, err := writer.Write(data)
 			if err != nil {
 				writeErr = err
diff --git a/weed/replication/sink/gcssink/gcs_sink.go b/weed/replication/sink/gcssink/gcs_sink.go
index b1a8d7753..bb5a54272 100644
--- a/weed/replication/sink/gcssink/gcs_sink.go
+++ b/weed/replication/sink/gcssink/gcs_sink.go
@@ -101,7 +101,7 @@ func (g *GcsSink) CreateEntry(key string, entry *filer_pb.Entry) error {
 			return err
 		}
 
-		err = util.ReadUrlAsStream(fileUrl, nil, false, chunk.IsFullChunk, chunk.Offset, int(chunk.Size), func(data []byte) {
+		err = util.ReadUrlAsStream(fileUrl, nil, false, chunk.IsFullChunk(), chunk.Offset, int(chunk.Size), func(data []byte) {
 			wc.Write(data)
 		})
 
diff --git a/weed/util/chunk_cache/chunk_cache.go b/weed/util/chunk_cache/chunk_cache.go
index 7c4a77304..232e57a55 100644
--- a/weed/util/chunk_cache/chunk_cache.go
+++ b/weed/util/chunk_cache/chunk_cache.go
@@ -1,52 +1,39 @@
 package chunk_cache
 
 import (
-	"fmt"
-	"path"
-	"sort"
 	"sync"
 
 	"github.com/chrislusf/seaweedfs/weed/glog"
-	"github.com/chrislusf/seaweedfs/weed/storage"
 	"github.com/chrislusf/seaweedfs/weed/storage/needle"
 )
 
+const (
+	memCacheSizeLimit = 1024 * 1024
+)
+
 // a global cache for recently accessed file chunks
 type ChunkCache struct {
-	memCache   *ChunkCacheInMemory
-	diskCaches []*ChunkCacheVolume
+	memCache  *ChunkCacheInMemory
+	diskCache *OnDiskCacheLayer
 	sync.RWMutex
 }
 
 func NewChunkCache(maxEntries int64, dir string, diskSizeMB int64, segmentCount int) *ChunkCache {
-	c := &ChunkCache{
-		memCache: NewChunkCacheInMemory(maxEntries),
-	}
 
 	volumeCount, volumeSize := int(diskSizeMB/30000), int64(30000)
 	if volumeCount < segmentCount {
 		volumeCount, volumeSize = segmentCount, diskSizeMB/int64(segmentCount)
 	}
 
-	for i := 0; i < volumeCount; i++ {
-		fileName := path.Join(dir, fmt.Sprintf("cache_%d", i))
-		diskCache, err := LoadOrCreateChunkCacheVolume(fileName, volumeSize*1024*1024)
-		if err != nil {
-			glog.Errorf("failed to add cache %s : %v", fileName, err)
-		} else {
-			c.diskCaches = append(c.diskCaches, diskCache)
-		}
+	c := &ChunkCache{
+		memCache:  NewChunkCacheInMemory(maxEntries),
+		diskCache: NewOnDiskCacheLayer(dir, "cache", volumeCount, volumeSize),
 	}
 
-	// keep newest cache to the front
-	sort.Slice(c.diskCaches, func(i, j int) bool {
-		return c.diskCaches[i].lastModTime.After(c.diskCaches[j].lastModTime)
-	})
-
 	return c
 }
 
-func (c *ChunkCache) GetChunk(fileId string) (data []byte) {
+func (c *ChunkCache) GetChunk(fileId string, chunkSize uint64) (data []byte) {
 	if c == nil {
 		return
 	}
@@ -54,12 +41,15 @@ func (c *ChunkCache) GetChunk(fileId string) (data []byte) {
 	c.RLock()
 	defer c.RUnlock()
 
-	return c.doGetChunk(fileId)
+	return c.doGetChunk(fileId, chunkSize)
 }
 
-func (c *ChunkCache) doGetChunk(fileId string) (data []byte) {
-	if data = c.memCache.GetChunk(fileId); data != nil {
-		return data
+func (c *ChunkCache) doGetChunk(fileId string, chunkSize uint64) (data []byte) {
+
+	if chunkSize < memCacheSizeLimit {
+		if data = c.memCache.GetChunk(fileId); data != nil {
+			return data
+		}
 	}
 
 	fid, err := needle.ParseFileIdFromString(fileId)
@@ -67,20 +57,9 @@ func (c *ChunkCache) doGetChunk(fileId string) (data []byte) {
 		glog.Errorf("failed to parse file id %s", fileId)
 		return nil
 	}
-	for _, diskCache := range c.diskCaches {
-		data, err = diskCache.GetNeedle(fid.Key)
-		if err == storage.ErrorNotFound {
-			continue
-		}
-		if err != nil {
-			glog.Errorf("failed to read cache file %s id %s", diskCache.fileName, fileId)
-			continue
-		}
-		if len(data) != 0 {
-			return
-		}
-	}
-	return nil
+
+	return c.diskCache.getChunk(fid.Key)
+
 }
 
 func (c *ChunkCache) SetChunk(fileId string, data []byte) {
@@ -95,22 +74,8 @@ func (c *ChunkCache) SetChunk(fileId string, data []byte) {
 
 func (c *ChunkCache) doSetChunk(fileId string, data []byte) {
 
-	c.memCache.SetChunk(fileId, data)
-
-	if len(c.diskCaches) == 0 {
-		return
-	}
-
-	if c.diskCaches[0].fileSize+int64(len(data)) > c.diskCaches[0].sizeLimit {
-		t, resetErr := c.diskCaches[len(c.diskCaches)-1].Reset()
-		if resetErr != nil {
-			glog.Errorf("failed to reset cache file %s", c.diskCaches[len(c.diskCaches)-1].fileName)
-			return
-		}
-		for i := len(c.diskCaches) - 1; i > 0; i-- {
-			c.diskCaches[i] = c.diskCaches[i-1]
-		}
-		c.diskCaches[0] = t
+	if len(data) < memCacheSizeLimit {
+		c.memCache.SetChunk(fileId, data)
 	}
 
 	fid, err := needle.ParseFileIdFromString(fileId)
@@ -118,7 +83,8 @@ func (c *ChunkCache) doSetChunk(fileId string, data []byte) {
 		glog.Errorf("failed to parse file id %s", fileId)
 		return
 	}
-	c.diskCaches[0].WriteNeedle(fid.Key, data)
+
+	c.diskCache.setChunk(fid.Key, data)
 
 }
 
@@ -128,7 +94,5 @@ func (c *ChunkCache) Shutdown() {
 	}
 	c.Lock()
 	defer c.Unlock()
-	for _, diskCache := range c.diskCaches {
-		diskCache.Shutdown()
-	}
+	c.diskCache.shutdown()
 }
diff --git a/weed/util/chunk_cache/chunk_cache_on_disk_test.go b/weed/util/chunk_cache/chunk_cache_on_disk_test.go
index f93daf5a7..63bcba2be 100644
--- a/weed/util/chunk_cache/chunk_cache_on_disk_test.go
+++ b/weed/util/chunk_cache/chunk_cache_on_disk_test.go
@@ -23,6 +23,7 @@ func TestOnDisk(t *testing.T) {
 	type test_data struct {
 		data   []byte
 		fileId string
+		size   uint64
 	}
 	testData := make([]*test_data, writeCount)
 	for i := 0; i < writeCount; i++ {
@@ -31,12 +32,13 @@ func TestOnDisk(t *testing.T) {
 		testData[i] = &test_data{
 			data:   buff,
 			fileId: fmt.Sprintf("1,%daabbccdd", i+1),
+			size:   uint64(len(buff)),
 		}
 		cache.SetChunk(testData[i].fileId, testData[i].data)
 	}
 
 	for i := 0; i < writeCount; i++ {
-		data := cache.GetChunk(testData[i].fileId)
+		data := cache.GetChunk(testData[i].fileId, testData[i].size)
 		if bytes.Compare(data, testData[i].data) != 0 {
 			t.Errorf("failed to write to and read from cache: %d", i)
 		}
@@ -47,7 +49,7 @@ func TestOnDisk(t *testing.T) {
 	cache = NewChunkCache(0, tmpDir, totalDiskSizeMb, segmentCount)
 
 	for i := 0; i < writeCount; i++ {
-		data := cache.GetChunk(testData[i].fileId)
+		data := cache.GetChunk(testData[i].fileId, testData[i].size)
 		if bytes.Compare(data, testData[i].data) != 0 {
 			t.Errorf("failed to write to and read from cache: %d", i)
 		}
diff --git a/weed/util/chunk_cache/on_disk_cache_layer.go b/weed/util/chunk_cache/on_disk_cache_layer.go
new file mode 100644
index 000000000..065188ac3
--- /dev/null
+++ b/weed/util/chunk_cache/on_disk_cache_layer.go
@@ -0,0 +1,83 @@
+package chunk_cache
+
+import (
+	"fmt"
+	"path"
+	"sort"
+
+	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/storage"
+	"github.com/chrislusf/seaweedfs/weed/storage/types"
+)
+
+type OnDiskCacheLayer struct {
+	diskCaches []*ChunkCacheVolume
+}
+
+func NewOnDiskCacheLayer(dir, namePrefix string, volumeCount int, volumeSize int64) *OnDiskCacheLayer{
+	c := &OnDiskCacheLayer{}
+	for i := 0; i < volumeCount; i++ {
+		fileName := path.Join(dir, fmt.Sprintf("%s_%d", namePrefix, i))
+		diskCache, err := LoadOrCreateChunkCacheVolume(fileName, volumeSize*1024*1024)
+		if err != nil {
+			glog.Errorf("failed to add cache %s : %v", fileName, err)
+		} else {
+			c.diskCaches = append(c.diskCaches, diskCache)
+		}
+	}
+
+	// keep newest cache to the front
+	sort.Slice(c.diskCaches, func(i, j int) bool {
+		return c.diskCaches[i].lastModTime.After(c.diskCaches[j].lastModTime)
+	})
+
+	return c
+}
+
+func (c *OnDiskCacheLayer) setChunk(needleId types.NeedleId, data []byte) {
+
+	if c.diskCaches[0].fileSize+int64(len(data)) > c.diskCaches[0].sizeLimit {
+		t, resetErr := c.diskCaches[len(c.diskCaches)-1].Reset()
+		if resetErr != nil {
+			glog.Errorf("failed to reset cache file %s", c.diskCaches[len(c.diskCaches)-1].fileName)
+			return
+		}
+		for i := len(c.diskCaches) - 1; i > 0; i-- {
+			c.diskCaches[i] = c.diskCaches[i-1]
+		}
+		c.diskCaches[0] = t
+	}
+
+	c.diskCaches[0].WriteNeedle(needleId, data)
+
+}
+
+func (c *OnDiskCacheLayer) getChunk(needleId types.NeedleId) (data []byte){
+
+	var err error
+
+	for _, diskCache := range c.diskCaches {
+		data, err = diskCache.GetNeedle(needleId)
+		if err == storage.ErrorNotFound {
+			continue
+		}
+		if err != nil {
+			glog.Errorf("failed to read cache file %s id %d", diskCache.fileName, needleId)
+			continue
+		}
+		if len(data) != 0 {
+			return
+		}
+	}
+
+	return nil
+
+}
+
+func (c *OnDiskCacheLayer) shutdown(){
+
+	for _, diskCache := range c.diskCaches {
+		diskCache.Shutdown()
+	}
+
+}

From 2b5c4fbbf37e25adfa19b081c4adf5458b05b66c Mon Sep 17 00:00:00 2001
From: Chris Lu <chris.lu@gmail.com>
Date: Mon, 13 Apr 2020 22:19:27 -0700
Subject: [PATCH 63/81] tiered caching

1/4 for small less than 1MB files. 1/4 for 1~4MB files, 1/2 for bigger than 4MB files
---
 weed/filesys/wfs.go                           |  2 +-
 weed/server/webdav_server.go                  |  2 +-
 weed/util/chunk_cache/chunk_cache.go          | 43 +++++++++++++------
 .../chunk_cache/chunk_cache_on_disk_test.go   |  7 ++-
 weed/util/chunk_cache/on_disk_cache_layer.go  |  8 +++-
 5 files changed, 41 insertions(+), 21 deletions(-)

diff --git a/weed/filesys/wfs.go b/weed/filesys/wfs.go
index 64c5d5aa6..b3772d683 100644
--- a/weed/filesys/wfs.go
+++ b/weed/filesys/wfs.go
@@ -84,7 +84,7 @@ func NewSeaweedFileSystem(option *Option) *WFS {
 		},
 	}
 	if option.CacheSizeMB > 0 {
-		wfs.chunkCache = chunk_cache.NewChunkCache(256, option.CacheDir, option.CacheSizeMB, 4)
+		wfs.chunkCache = chunk_cache.NewChunkCache(256, option.CacheDir, option.CacheSizeMB)
 		util.OnInterrupt(func() {
 			wfs.chunkCache.Shutdown()
 		})
diff --git a/weed/server/webdav_server.go b/weed/server/webdav_server.go
index affc953bc..445cc7b4d 100644
--- a/weed/server/webdav_server.go
+++ b/weed/server/webdav_server.go
@@ -99,7 +99,7 @@ type WebDavFile struct {
 
 func NewWebDavFileSystem(option *WebDavOption) (webdav.FileSystem, error) {
 
-	chunkCache := chunk_cache.NewChunkCache(256, option.CacheDir, option.CacheSizeMB, 4)
+	chunkCache := chunk_cache.NewChunkCache(256, option.CacheDir, option.CacheSizeMB)
 	util.OnInterrupt(func() {
 		chunkCache.Shutdown()
 	})
diff --git a/weed/util/chunk_cache/chunk_cache.go b/weed/util/chunk_cache/chunk_cache.go
index 232e57a55..e1d4b639f 100644
--- a/weed/util/chunk_cache/chunk_cache.go
+++ b/weed/util/chunk_cache/chunk_cache.go
@@ -8,27 +8,27 @@ import (
 )
 
 const (
-	memCacheSizeLimit = 1024 * 1024
+	memCacheSizeLimit     = 1024 * 1024
+	onDiskCacheSizeLimit0 = memCacheSizeLimit
+	onDiskCacheSizeLimit1 = 4 * memCacheSizeLimit
 )
 
 // a global cache for recently accessed file chunks
 type ChunkCache struct {
-	memCache  *ChunkCacheInMemory
-	diskCache *OnDiskCacheLayer
+	memCache   *ChunkCacheInMemory
+	diskCaches []*OnDiskCacheLayer
 	sync.RWMutex
 }
 
-func NewChunkCache(maxEntries int64, dir string, diskSizeMB int64, segmentCount int) *ChunkCache {
-
-	volumeCount, volumeSize := int(diskSizeMB/30000), int64(30000)
-	if volumeCount < segmentCount {
-		volumeCount, volumeSize = segmentCount, diskSizeMB/int64(segmentCount)
-	}
+func NewChunkCache(maxEntries int64, dir string, diskSizeMB int64) *ChunkCache {
 
 	c := &ChunkCache{
-		memCache:  NewChunkCacheInMemory(maxEntries),
-		diskCache: NewOnDiskCacheLayer(dir, "cache", volumeCount, volumeSize),
+		memCache: NewChunkCacheInMemory(maxEntries),
 	}
+	c.diskCaches = make([]*OnDiskCacheLayer, 3)
+	c.diskCaches[0] = NewOnDiskCacheLayer(dir, "c0_1", diskSizeMB/4, 4)
+	c.diskCaches[1] = NewOnDiskCacheLayer(dir, "c1_4", diskSizeMB/4, 4)
+	c.diskCaches[2] = NewOnDiskCacheLayer(dir, "cache", diskSizeMB/2, 4)
 
 	return c
 }
@@ -58,7 +58,14 @@ func (c *ChunkCache) doGetChunk(fileId string, chunkSize uint64) (data []byte) {
 		return nil
 	}
 
-	return c.diskCache.getChunk(fid.Key)
+	for _, diskCache := range c.diskCaches {
+		data := diskCache.getChunk(fid.Key)
+		if len(data) != 0 {
+			return data
+		}
+	}
+
+	return nil
 
 }
 
@@ -84,7 +91,13 @@ func (c *ChunkCache) doSetChunk(fileId string, data []byte) {
 		return
 	}
 
-	c.diskCache.setChunk(fid.Key, data)
+	if len(data) < onDiskCacheSizeLimit0 {
+		c.diskCaches[0].setChunk(fid.Key, data)
+	} else if len(data) < onDiskCacheSizeLimit1 {
+		c.diskCaches[1].setChunk(fid.Key, data)
+	} else {
+		c.diskCaches[2].setChunk(fid.Key, data)
+	}
 
 }
 
@@ -94,5 +107,7 @@ func (c *ChunkCache) Shutdown() {
 	}
 	c.Lock()
 	defer c.Unlock()
-	c.diskCache.shutdown()
+	for _, diskCache := range c.diskCaches {
+		diskCache.shutdown()
+	}
 }
diff --git a/weed/util/chunk_cache/chunk_cache_on_disk_test.go b/weed/util/chunk_cache/chunk_cache_on_disk_test.go
index 63bcba2be..f061f2ba2 100644
--- a/weed/util/chunk_cache/chunk_cache_on_disk_test.go
+++ b/weed/util/chunk_cache/chunk_cache_on_disk_test.go
@@ -14,10 +14,9 @@ func TestOnDisk(t *testing.T) {
 	tmpDir, _ := ioutil.TempDir("", "c")
 	defer os.RemoveAll(tmpDir)
 
-	totalDiskSizeMb := int64(6)
-	segmentCount := 2
+	totalDiskSizeMb := int64(32)
 
-	cache := NewChunkCache(0, tmpDir, totalDiskSizeMb, segmentCount)
+	cache := NewChunkCache(0, tmpDir, totalDiskSizeMb)
 
 	writeCount := 5
 	type test_data struct {
@@ -46,7 +45,7 @@ func TestOnDisk(t *testing.T) {
 
 	cache.Shutdown()
 
-	cache = NewChunkCache(0, tmpDir, totalDiskSizeMb, segmentCount)
+	cache = NewChunkCache(0, tmpDir, totalDiskSizeMb)
 
 	for i := 0; i < writeCount; i++ {
 		data := cache.GetChunk(testData[i].fileId, testData[i].size)
diff --git a/weed/util/chunk_cache/on_disk_cache_layer.go b/weed/util/chunk_cache/on_disk_cache_layer.go
index 065188ac3..9bd9c2b44 100644
--- a/weed/util/chunk_cache/on_disk_cache_layer.go
+++ b/weed/util/chunk_cache/on_disk_cache_layer.go
@@ -14,7 +14,13 @@ type OnDiskCacheLayer struct {
 	diskCaches []*ChunkCacheVolume
 }
 
-func NewOnDiskCacheLayer(dir, namePrefix string, volumeCount int, volumeSize int64) *OnDiskCacheLayer{
+func NewOnDiskCacheLayer(dir, namePrefix string, diskSizeMB int64, segmentCount int) *OnDiskCacheLayer{
+
+	volumeCount, volumeSize := int(diskSizeMB/30000), int64(30000)
+	if volumeCount < segmentCount {
+		volumeCount, volumeSize = segmentCount, diskSizeMB/int64(segmentCount)
+	}
+
 	c := &OnDiskCacheLayer{}
 	for i := 0; i < volumeCount; i++ {
 		fileName := path.Join(dir, fmt.Sprintf("%s_%d", namePrefix, i))

From d2d15978502ff3b45b010f546beaf226321dddb1 Mon Sep 17 00:00:00 2001
From: Chris Lu <chris.lu@gmail.com>
Date: Tue, 14 Apr 2020 10:52:53 -0700
Subject: [PATCH 64/81] volume: detect gzip effectiveness only when content
 type is empty

revert part of https://github.com/chrislusf/seaweedfs/commit/2286eda575861cceb4464bf677e9885e069de385
---
 weed/operation/upload_content.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/weed/operation/upload_content.go b/weed/operation/upload_content.go
index f6f3d0dcc..f582c4279 100644
--- a/weed/operation/upload_content.go
+++ b/weed/operation/upload_content.go
@@ -79,7 +79,7 @@ func doUploadData(uploadUrl string, filename string, cipher bool, data []byte, i
 	if !isInputGzipped {
 		if shouldBeZipped, iAmSure := util.IsGzippableFileType(filepath.Base(filename), mtype); iAmSure && shouldBeZipped {
 			shouldGzipNow = true
-		} else if len(data) > 128 {
+		} else if mtype == "" && len(data) > 128 {
 			var compressed []byte
 			compressed, err = util.GzipData(data[0:128])
 			shouldGzipNow = len(compressed)*10 < 128*9 // can not compress to less than 90%

From 3f1d79512fbd2ff4c108af5016b38533e4b0300b Mon Sep 17 00:00:00 2001
From: Chris Lu <chris.lu@gmail.com>
Date: Tue, 14 Apr 2020 11:02:05 -0700
Subject: [PATCH 65/81] only detect when not sure about the types

---
 weed/operation/upload_content.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/weed/operation/upload_content.go b/weed/operation/upload_content.go
index f582c4279..d4a334f05 100644
--- a/weed/operation/upload_content.go
+++ b/weed/operation/upload_content.go
@@ -79,7 +79,7 @@ func doUploadData(uploadUrl string, filename string, cipher bool, data []byte, i
 	if !isInputGzipped {
 		if shouldBeZipped, iAmSure := util.IsGzippableFileType(filepath.Base(filename), mtype); iAmSure && shouldBeZipped {
 			shouldGzipNow = true
-		} else if mtype == "" && len(data) > 128 {
+		} else if !iAmSure && mtype == "" && len(data) > 128 {
 			var compressed []byte
 			compressed, err = util.GzipData(data[0:128])
 			shouldGzipNow = len(compressed)*10 < 128*9 // can not compress to less than 90%

From 9cacaf5eca3de4bb8b25d2dc61da4ae4429241cd Mon Sep 17 00:00:00 2001
From: Chris Lu <chris.lu@gmail.com>
Date: Tue, 14 Apr 2020 11:32:31 -0700
Subject: [PATCH 66/81] better detect file mime type

---
 weed/filesys/filehandle.go       | 12 ++----------
 weed/operation/upload_content.go |  6 ++++++
 2 files changed, 8 insertions(+), 10 deletions(-)

diff --git a/weed/filesys/filehandle.go b/weed/filesys/filehandle.go
index 935f0b314..d2983d53f 100644
--- a/weed/filesys/filehandle.go
+++ b/weed/filesys/filehandle.go
@@ -4,12 +4,9 @@ import (
 	"context"
 	"fmt"
 	"math"
-	"mime"
-	"path"
+	"net/http"
 	"time"
 
-	"github.com/gabriel-vasile/mimetype"
-
 	"github.com/chrislusf/seaweedfs/weed/filer2"
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
@@ -130,12 +127,7 @@ func (fh *FileHandle) Write(ctx context.Context, req *fuse.WriteRequest, resp *f
 
 	if req.Offset == 0 {
 		// detect mime type
-		detectedMIME := mimetype.Detect(data)
-		fh.contentType = detectedMIME.String()
-		if ext := path.Ext(fh.f.Name); ext != detectedMIME.Extension() {
-			fh.contentType = mime.TypeByExtension(ext)
-		}
-
+		fh.contentType = http.DetectContentType(data)
 		fh.dirtyMetadata = true
 	}
 
diff --git a/weed/operation/upload_content.go b/weed/operation/upload_content.go
index d4a334f05..5b0441ff9 100644
--- a/weed/operation/upload_content.go
+++ b/weed/operation/upload_content.go
@@ -77,6 +77,12 @@ func doUploadData(uploadUrl string, filename string, cipher bool, data []byte, i
 	contentIsGzipped := isInputGzipped
 	shouldGzipNow := false
 	if !isInputGzipped {
+		if mtype == "" {
+			mtype = http.DetectContentType(data)
+			if mtype == "application/octet-stream" {
+				mtype = ""
+			}
+		}
 		if shouldBeZipped, iAmSure := util.IsGzippableFileType(filepath.Base(filename), mtype); iAmSure && shouldBeZipped {
 			shouldGzipNow = true
 		} else if !iAmSure && mtype == "" && len(data) > 128 {

From aada5b9aa7fa0d38acd1cb719fe5af5c465f5b8f Mon Sep 17 00:00:00 2001
From: Reed <admin@mrwang.pw>
Date: Thu, 16 Apr 2020 10:08:08 +0800
Subject: [PATCH 67/81] making volume stats size info readable.

---
 weed/server/volume_server_ui/templates.go | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/weed/server/volume_server_ui/templates.go b/weed/server/volume_server_ui/templates.go
index 1c1394369..a3175e9ca 100644
--- a/weed/server/volume_server_ui/templates.go
+++ b/weed/server/volume_server_ui/templates.go
@@ -151,9 +151,9 @@ var StatusTpl = template.Must(template.New("status").Funcs(funcMap).Parse(`<!DOC
             <tr>
               <td><code>{{ .Id }}</code></td>
               <td>{{ .Collection }}</td>
-              <td>{{ .Size }} Bytes</td>
+              <td>{{ bytesToHumanReadble .Size }}</td>
               <td>{{ .FileCount }}</td>
-              <td>{{ .DeleteCount }} / {{.DeletedByteCount}} Bytes</td>
+              <td>{{ .DeleteCount }} / {{bytesToHumanReadble .DeletedByteCount}}</td>
               <td>{{ .Ttl }}</td>
               <td>{{ .ReadOnly }}</td>
             </tr>
@@ -181,9 +181,9 @@ var StatusTpl = template.Must(template.New("status").Funcs(funcMap).Parse(`<!DOC
             <tr>
               <td><code>{{ .Id }}</code></td>
               <td>{{ .Collection }}</td>
-              <td>{{ .Size }} Bytes</td>
+              <td>{{ bytesToHumanReadble .Size }}</td>
               <td>{{ .FileCount }}</td>
-              <td>{{ .DeleteCount }} / {{.DeletedByteCount}} Bytes</td>
+              <td>{{ .DeleteCount }} / {{bytesToHumanReadble .DeletedByteCount}}</td>
               <td>{{ .RemoteStorageName }}</td>
               <td>{{ .RemoteStorageKey }}</td>
             </tr>
@@ -209,7 +209,7 @@ var StatusTpl = template.Must(template.New("status").Funcs(funcMap).Parse(`<!DOC
             <tr>
               <td><code>{{ .VolumeId }}</code></td>
               <td>{{ .Collection }}</td>
-              <td>{{ .ShardSize }} Bytes</td>
+              <td>{{ bytesToHumanReadble .ShardSize }}</td>
               <td>{{ .ShardIdList }}</td>
               <td>{{ .CreatedAt.Format "02 Jan 06 15:04 -0700" }}</td>
             </tr>

From ce4b369be25f61be0810e2bc119964b613cca121 Mon Sep 17 00:00:00 2001
From: Chris Lu <chris.lu@gmail.com>
Date: Thu, 16 Apr 2020 02:21:23 -0700
Subject: [PATCH 68/81] scaffolding messaging

---
 weed/command/msg_broker.go            |   6 +-
 weed/pb/Makefile                      |   2 +-
 weed/pb/messaging.proto               |  76 ++++
 weed/pb/messaging_pb/messaging.pb.go  | 601 ++++++++++++++++++++++++++
 weed/pb/queue.proto                   |  66 ---
 weed/pb/queue_pb/queue.pb.go          | 516 ----------------------
 weed/server/msg_broker_grpc_server.go |  10 +-
 7 files changed, 685 insertions(+), 592 deletions(-)
 create mode 100644 weed/pb/messaging.proto
 create mode 100644 weed/pb/messaging_pb/messaging.pb.go
 delete mode 100644 weed/pb/queue.proto
 delete mode 100644 weed/pb/queue_pb/queue.pb.go

diff --git a/weed/command/msg_broker.go b/weed/command/msg_broker.go
index 093cff4a6..3cb424298 100644
--- a/weed/command/msg_broker.go
+++ b/weed/command/msg_broker.go
@@ -10,7 +10,7 @@ import (
 
 	"github.com/chrislusf/seaweedfs/weed/pb"
 	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
-	"github.com/chrislusf/seaweedfs/weed/pb/queue_pb"
+	"github.com/chrislusf/seaweedfs/weed/pb/messaging_pb"
 	"github.com/chrislusf/seaweedfs/weed/security"
 	weed_server "github.com/chrislusf/seaweedfs/weed/server"
 
@@ -25,14 +25,12 @@ var (
 type QueueOptions struct {
 	filer      *string
 	port       *int
-	defaultTtl *string
 }
 
 func init() {
 	cmdMsgBroker.Run = runMsgBroker // break init cycle
 	messageBrokerStandaloneOptions.filer = cmdMsgBroker.Flag.String("filer", "localhost:8888", "filer server address")
 	messageBrokerStandaloneOptions.port = cmdMsgBroker.Flag.Int("port", 17777, "queue server gRPC listen port")
-	messageBrokerStandaloneOptions.defaultTtl = cmdMsgBroker.Flag.String("ttl", "1h", "time to live, e.g.: 1m, 1h, 1d, 1M, 1y")
 }
 
 var cmdMsgBroker = &Command{
@@ -94,7 +92,7 @@ func (msgBrokerOpt *QueueOptions) startQueueServer() bool {
 		glog.Fatalf("failed to listen on grpc port %d: %v", *msgBrokerOpt.port, err)
 	}
 	grpcS := pb.NewGrpcServer(security.LoadServerTLS(util.GetViper(), "grpc.msg_broker"))
-	queue_pb.RegisterSeaweedQueueServer(grpcS, qs)
+	messaging_pb.RegisterSeaweedMessagingServer(grpcS, qs)
 	reflection.Register(grpcS)
 	grpcS.Serve(grpcL)
 
diff --git a/weed/pb/Makefile b/weed/pb/Makefile
index 6680b7ca2..5053669d8 100644
--- a/weed/pb/Makefile
+++ b/weed/pb/Makefile
@@ -7,6 +7,6 @@ gen:
 	protoc volume_server.proto --go_out=plugins=grpc:./volume_server_pb
 	protoc filer.proto --go_out=plugins=grpc:./filer_pb
 	protoc iam.proto --go_out=plugins=grpc:./iam_pb
-	protoc queue.proto --go_out=plugins=grpc:./queue_pb
+	protoc messaging.proto --go_out=plugins=grpc:./messaging_pb
 	# protoc filer.proto --java_out=../../other/java/client/src/main/java
 	cp filer.proto ../../other/java/client/src/main/proto
diff --git a/weed/pb/messaging.proto b/weed/pb/messaging.proto
new file mode 100644
index 000000000..050c6fb17
--- /dev/null
+++ b/weed/pb/messaging.proto
@@ -0,0 +1,76 @@
+syntax = "proto3";
+
+package messaging_pb;
+
+option java_package = "seaweedfs.client";
+option java_outer_classname = "MessagingProto";
+
+//////////////////////////////////////////////////
+
+service SeaweedMessaging {
+
+    rpc Subscribe (SubscribeRequest) returns (stream Message) {
+    }
+
+    rpc Publish (stream PublishRequest) returns (stream PublishResponse) {
+    }
+
+    rpc ConfigureTopic (ConfigureTopicRequest) returns (ConfigureTopicResponse) {
+    }
+
+    rpc GetTopicConfiguration (GetTopicConfigurationRequest) returns (GetTopicConfigurationResponse) {
+    }
+
+}
+
+//////////////////////////////////////////////////
+
+message SubscribeRequest {
+    string namespace = 1;
+    string topic = 2;
+    int32 partition = 3;
+    enum StartPosition {
+        LATEST = 0; // Start at the newest message
+        EARLIEST = 1; // Start at the oldest message
+        TIMESTAMP = 2; // Start after a specified timestamp, exclusive
+    }
+    StartPosition startPosition = 4; // Where to begin consuming from
+    int64 timestampNs = 5; // timestamp in nano seconds
+}
+
+message Message {
+    int64 timestamp = 1 [jstype = JS_STRING]; // When the message was received by the broker
+    bytes key = 2; // Message key
+    bytes value = 3; // Message payload
+    map<string, bytes> headers = 4; // Message headers
+}
+
+message PublishRequest {
+    string namespace = 1; // only needed on the initial request
+    string topic = 2; // only needed on the initial request
+    int32 partition = 4;
+    bytes key = 5; // Message key
+    bytes value = 6; // Message payload
+    map<string, bytes> headers = 7; // Message headers
+}
+
+message PublishResponse {
+    int32 partition_count = 1;
+}
+
+message ConfigureTopicRequest {
+    string namespace = 1;
+    string topic = 2;
+    int32 partition_count = 3;
+    string collection = 4;
+}
+message ConfigureTopicResponse {
+}
+
+message GetTopicConfigurationRequest {
+    string namespace = 1;
+    string topic = 2;
+}
+message GetTopicConfigurationResponse {
+    int32 partitions = 3;
+}
diff --git a/weed/pb/messaging_pb/messaging.pb.go b/weed/pb/messaging_pb/messaging.pb.go
new file mode 100644
index 000000000..64fc7c20b
--- /dev/null
+++ b/weed/pb/messaging_pb/messaging.pb.go
@@ -0,0 +1,601 @@
+// Code generated by protoc-gen-go.
+// source: messaging.proto
+// DO NOT EDIT!
+
+/*
+Package messaging_pb is a generated protocol buffer package.
+
+It is generated from these files:
+	messaging.proto
+
+It has these top-level messages:
+	SubscribeRequest
+	Message
+	PublishRequest
+	PublishResponse
+	ConfigureTopicRequest
+	ConfigureTopicResponse
+	GetTopicConfigurationRequest
+	GetTopicConfigurationResponse
+*/
+package messaging_pb
+
+import proto "github.com/golang/protobuf/proto"
+import fmt "fmt"
+import math "math"
+
+import (
+	context "golang.org/x/net/context"
+	grpc "google.golang.org/grpc"
+)
+
+// Reference imports to suppress errors if they are not otherwise used.
+var _ = proto.Marshal
+var _ = fmt.Errorf
+var _ = math.Inf
+
+// This is a compile-time assertion to ensure that this generated file
+// is compatible with the proto package it is being compiled against.
+// A compilation error at this line likely means your copy of the
+// proto package needs to be updated.
+const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package
+
+type SubscribeRequest_StartPosition int32
+
+const (
+	SubscribeRequest_LATEST    SubscribeRequest_StartPosition = 0
+	SubscribeRequest_EARLIEST  SubscribeRequest_StartPosition = 1
+	SubscribeRequest_TIMESTAMP SubscribeRequest_StartPosition = 2
+)
+
+var SubscribeRequest_StartPosition_name = map[int32]string{
+	0: "LATEST",
+	1: "EARLIEST",
+	2: "TIMESTAMP",
+}
+var SubscribeRequest_StartPosition_value = map[string]int32{
+	"LATEST":    0,
+	"EARLIEST":  1,
+	"TIMESTAMP": 2,
+}
+
+func (x SubscribeRequest_StartPosition) String() string {
+	return proto.EnumName(SubscribeRequest_StartPosition_name, int32(x))
+}
+func (SubscribeRequest_StartPosition) EnumDescriptor() ([]byte, []int) {
+	return fileDescriptor0, []int{0, 0}
+}
+
+type SubscribeRequest struct {
+	Namespace     string                         `protobuf:"bytes,1,opt,name=namespace" json:"namespace,omitempty"`
+	Topic         string                         `protobuf:"bytes,2,opt,name=topic" json:"topic,omitempty"`
+	Partition     int32                          `protobuf:"varint,3,opt,name=partition" json:"partition,omitempty"`
+	StartPosition SubscribeRequest_StartPosition `protobuf:"varint,4,opt,name=startPosition,enum=messaging_pb.SubscribeRequest_StartPosition" json:"startPosition,omitempty"`
+	TimestampNs   int64                          `protobuf:"varint,5,opt,name=timestampNs" json:"timestampNs,omitempty"`
+}
+
+func (m *SubscribeRequest) Reset()                    { *m = SubscribeRequest{} }
+func (m *SubscribeRequest) String() string            { return proto.CompactTextString(m) }
+func (*SubscribeRequest) ProtoMessage()               {}
+func (*SubscribeRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{0} }
+
+func (m *SubscribeRequest) GetNamespace() string {
+	if m != nil {
+		return m.Namespace
+	}
+	return ""
+}
+
+func (m *SubscribeRequest) GetTopic() string {
+	if m != nil {
+		return m.Topic
+	}
+	return ""
+}
+
+func (m *SubscribeRequest) GetPartition() int32 {
+	if m != nil {
+		return m.Partition
+	}
+	return 0
+}
+
+func (m *SubscribeRequest) GetStartPosition() SubscribeRequest_StartPosition {
+	if m != nil {
+		return m.StartPosition
+	}
+	return SubscribeRequest_LATEST
+}
+
+func (m *SubscribeRequest) GetTimestampNs() int64 {
+	if m != nil {
+		return m.TimestampNs
+	}
+	return 0
+}
+
+type Message struct {
+	Timestamp int64             `protobuf:"varint,1,opt,name=timestamp" json:"timestamp,omitempty"`
+	Key       []byte            `protobuf:"bytes,2,opt,name=key,proto3" json:"key,omitempty"`
+	Value     []byte            `protobuf:"bytes,3,opt,name=value,proto3" json:"value,omitempty"`
+	Headers   map[string][]byte `protobuf:"bytes,4,rep,name=headers" json:"headers,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value,proto3"`
+}
+
+func (m *Message) Reset()                    { *m = Message{} }
+func (m *Message) String() string            { return proto.CompactTextString(m) }
+func (*Message) ProtoMessage()               {}
+func (*Message) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{1} }
+
+func (m *Message) GetTimestamp() int64 {
+	if m != nil {
+		return m.Timestamp
+	}
+	return 0
+}
+
+func (m *Message) GetKey() []byte {
+	if m != nil {
+		return m.Key
+	}
+	return nil
+}
+
+func (m *Message) GetValue() []byte {
+	if m != nil {
+		return m.Value
+	}
+	return nil
+}
+
+func (m *Message) GetHeaders() map[string][]byte {
+	if m != nil {
+		return m.Headers
+	}
+	return nil
+}
+
+type PublishRequest struct {
+	Namespace string            `protobuf:"bytes,1,opt,name=namespace" json:"namespace,omitempty"`
+	Topic     string            `protobuf:"bytes,2,opt,name=topic" json:"topic,omitempty"`
+	Partition int32             `protobuf:"varint,4,opt,name=partition" json:"partition,omitempty"`
+	Key       []byte            `protobuf:"bytes,5,opt,name=key,proto3" json:"key,omitempty"`
+	Value     []byte            `protobuf:"bytes,6,opt,name=value,proto3" json:"value,omitempty"`
+	Headers   map[string][]byte `protobuf:"bytes,7,rep,name=headers" json:"headers,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value,proto3"`
+}
+
+func (m *PublishRequest) Reset()                    { *m = PublishRequest{} }
+func (m *PublishRequest) String() string            { return proto.CompactTextString(m) }
+func (*PublishRequest) ProtoMessage()               {}
+func (*PublishRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{2} }
+
+func (m *PublishRequest) GetNamespace() string {
+	if m != nil {
+		return m.Namespace
+	}
+	return ""
+}
+
+func (m *PublishRequest) GetTopic() string {
+	if m != nil {
+		return m.Topic
+	}
+	return ""
+}
+
+func (m *PublishRequest) GetPartition() int32 {
+	if m != nil {
+		return m.Partition
+	}
+	return 0
+}
+
+func (m *PublishRequest) GetKey() []byte {
+	if m != nil {
+		return m.Key
+	}
+	return nil
+}
+
+func (m *PublishRequest) GetValue() []byte {
+	if m != nil {
+		return m.Value
+	}
+	return nil
+}
+
+func (m *PublishRequest) GetHeaders() map[string][]byte {
+	if m != nil {
+		return m.Headers
+	}
+	return nil
+}
+
+type PublishResponse struct {
+	PartitionCount int32 `protobuf:"varint,1,opt,name=partition_count,json=partitionCount" json:"partition_count,omitempty"`
+}
+
+func (m *PublishResponse) Reset()                    { *m = PublishResponse{} }
+func (m *PublishResponse) String() string            { return proto.CompactTextString(m) }
+func (*PublishResponse) ProtoMessage()               {}
+func (*PublishResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{3} }
+
+func (m *PublishResponse) GetPartitionCount() int32 {
+	if m != nil {
+		return m.PartitionCount
+	}
+	return 0
+}
+
+type ConfigureTopicRequest struct {
+	Namespace      string `protobuf:"bytes,1,opt,name=namespace" json:"namespace,omitempty"`
+	Topic          string `protobuf:"bytes,2,opt,name=topic" json:"topic,omitempty"`
+	PartitionCount int32  `protobuf:"varint,3,opt,name=partition_count,json=partitionCount" json:"partition_count,omitempty"`
+	Collection     string `protobuf:"bytes,4,opt,name=collection" json:"collection,omitempty"`
+}
+
+func (m *ConfigureTopicRequest) Reset()                    { *m = ConfigureTopicRequest{} }
+func (m *ConfigureTopicRequest) String() string            { return proto.CompactTextString(m) }
+func (*ConfigureTopicRequest) ProtoMessage()               {}
+func (*ConfigureTopicRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{4} }
+
+func (m *ConfigureTopicRequest) GetNamespace() string {
+	if m != nil {
+		return m.Namespace
+	}
+	return ""
+}
+
+func (m *ConfigureTopicRequest) GetTopic() string {
+	if m != nil {
+		return m.Topic
+	}
+	return ""
+}
+
+func (m *ConfigureTopicRequest) GetPartitionCount() int32 {
+	if m != nil {
+		return m.PartitionCount
+	}
+	return 0
+}
+
+func (m *ConfigureTopicRequest) GetCollection() string {
+	if m != nil {
+		return m.Collection
+	}
+	return ""
+}
+
+type ConfigureTopicResponse struct {
+}
+
+func (m *ConfigureTopicResponse) Reset()                    { *m = ConfigureTopicResponse{} }
+func (m *ConfigureTopicResponse) String() string            { return proto.CompactTextString(m) }
+func (*ConfigureTopicResponse) ProtoMessage()               {}
+func (*ConfigureTopicResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{5} }
+
+type GetTopicConfigurationRequest struct {
+	Namespace string `protobuf:"bytes,1,opt,name=namespace" json:"namespace,omitempty"`
+	Topic     string `protobuf:"bytes,2,opt,name=topic" json:"topic,omitempty"`
+}
+
+func (m *GetTopicConfigurationRequest) Reset()                    { *m = GetTopicConfigurationRequest{} }
+func (m *GetTopicConfigurationRequest) String() string            { return proto.CompactTextString(m) }
+func (*GetTopicConfigurationRequest) ProtoMessage()               {}
+func (*GetTopicConfigurationRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{6} }
+
+func (m *GetTopicConfigurationRequest) GetNamespace() string {
+	if m != nil {
+		return m.Namespace
+	}
+	return ""
+}
+
+func (m *GetTopicConfigurationRequest) GetTopic() string {
+	if m != nil {
+		return m.Topic
+	}
+	return ""
+}
+
+type GetTopicConfigurationResponse struct {
+	Partitions int32 `protobuf:"varint,3,opt,name=partitions" json:"partitions,omitempty"`
+}
+
+func (m *GetTopicConfigurationResponse) Reset()                    { *m = GetTopicConfigurationResponse{} }
+func (m *GetTopicConfigurationResponse) String() string            { return proto.CompactTextString(m) }
+func (*GetTopicConfigurationResponse) ProtoMessage()               {}
+func (*GetTopicConfigurationResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{7} }
+
+func (m *GetTopicConfigurationResponse) GetPartitions() int32 {
+	if m != nil {
+		return m.Partitions
+	}
+	return 0
+}
+
+func init() {
+	proto.RegisterType((*SubscribeRequest)(nil), "messaging_pb.SubscribeRequest")
+	proto.RegisterType((*Message)(nil), "messaging_pb.Message")
+	proto.RegisterType((*PublishRequest)(nil), "messaging_pb.PublishRequest")
+	proto.RegisterType((*PublishResponse)(nil), "messaging_pb.PublishResponse")
+	proto.RegisterType((*ConfigureTopicRequest)(nil), "messaging_pb.ConfigureTopicRequest")
+	proto.RegisterType((*ConfigureTopicResponse)(nil), "messaging_pb.ConfigureTopicResponse")
+	proto.RegisterType((*GetTopicConfigurationRequest)(nil), "messaging_pb.GetTopicConfigurationRequest")
+	proto.RegisterType((*GetTopicConfigurationResponse)(nil), "messaging_pb.GetTopicConfigurationResponse")
+	proto.RegisterEnum("messaging_pb.SubscribeRequest_StartPosition", SubscribeRequest_StartPosition_name, SubscribeRequest_StartPosition_value)
+}
+
+// Reference imports to suppress errors if they are not otherwise used.
+var _ context.Context
+var _ grpc.ClientConn
+
+// This is a compile-time assertion to ensure that this generated file
+// is compatible with the grpc package it is being compiled against.
+const _ = grpc.SupportPackageIsVersion4
+
+// Client API for SeaweedMessaging service
+
+type SeaweedMessagingClient interface {
+	Subscribe(ctx context.Context, in *SubscribeRequest, opts ...grpc.CallOption) (SeaweedMessaging_SubscribeClient, error)
+	Publish(ctx context.Context, opts ...grpc.CallOption) (SeaweedMessaging_PublishClient, error)
+	ConfigureTopic(ctx context.Context, in *ConfigureTopicRequest, opts ...grpc.CallOption) (*ConfigureTopicResponse, error)
+	GetTopicConfiguration(ctx context.Context, in *GetTopicConfigurationRequest, opts ...grpc.CallOption) (*GetTopicConfigurationResponse, error)
+}
+
+type seaweedMessagingClient struct {
+	cc *grpc.ClientConn
+}
+
+func NewSeaweedMessagingClient(cc *grpc.ClientConn) SeaweedMessagingClient {
+	return &seaweedMessagingClient{cc}
+}
+
+func (c *seaweedMessagingClient) Subscribe(ctx context.Context, in *SubscribeRequest, opts ...grpc.CallOption) (SeaweedMessaging_SubscribeClient, error) {
+	stream, err := grpc.NewClientStream(ctx, &_SeaweedMessaging_serviceDesc.Streams[0], c.cc, "/messaging_pb.SeaweedMessaging/Subscribe", opts...)
+	if err != nil {
+		return nil, err
+	}
+	x := &seaweedMessagingSubscribeClient{stream}
+	if err := x.ClientStream.SendMsg(in); err != nil {
+		return nil, err
+	}
+	if err := x.ClientStream.CloseSend(); err != nil {
+		return nil, err
+	}
+	return x, nil
+}
+
+type SeaweedMessaging_SubscribeClient interface {
+	Recv() (*Message, error)
+	grpc.ClientStream
+}
+
+type seaweedMessagingSubscribeClient struct {
+	grpc.ClientStream
+}
+
+func (x *seaweedMessagingSubscribeClient) Recv() (*Message, error) {
+	m := new(Message)
+	if err := x.ClientStream.RecvMsg(m); err != nil {
+		return nil, err
+	}
+	return m, nil
+}
+
+func (c *seaweedMessagingClient) Publish(ctx context.Context, opts ...grpc.CallOption) (SeaweedMessaging_PublishClient, error) {
+	stream, err := grpc.NewClientStream(ctx, &_SeaweedMessaging_serviceDesc.Streams[1], c.cc, "/messaging_pb.SeaweedMessaging/Publish", opts...)
+	if err != nil {
+		return nil, err
+	}
+	x := &seaweedMessagingPublishClient{stream}
+	return x, nil
+}
+
+type SeaweedMessaging_PublishClient interface {
+	Send(*PublishRequest) error
+	Recv() (*PublishResponse, error)
+	grpc.ClientStream
+}
+
+type seaweedMessagingPublishClient struct {
+	grpc.ClientStream
+}
+
+func (x *seaweedMessagingPublishClient) Send(m *PublishRequest) error {
+	return x.ClientStream.SendMsg(m)
+}
+
+func (x *seaweedMessagingPublishClient) Recv() (*PublishResponse, error) {
+	m := new(PublishResponse)
+	if err := x.ClientStream.RecvMsg(m); err != nil {
+		return nil, err
+	}
+	return m, nil
+}
+
+func (c *seaweedMessagingClient) ConfigureTopic(ctx context.Context, in *ConfigureTopicRequest, opts ...grpc.CallOption) (*ConfigureTopicResponse, error) {
+	out := new(ConfigureTopicResponse)
+	err := grpc.Invoke(ctx, "/messaging_pb.SeaweedMessaging/ConfigureTopic", in, out, c.cc, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *seaweedMessagingClient) GetTopicConfiguration(ctx context.Context, in *GetTopicConfigurationRequest, opts ...grpc.CallOption) (*GetTopicConfigurationResponse, error) {
+	out := new(GetTopicConfigurationResponse)
+	err := grpc.Invoke(ctx, "/messaging_pb.SeaweedMessaging/GetTopicConfiguration", in, out, c.cc, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+// Server API for SeaweedMessaging service
+
+type SeaweedMessagingServer interface {
+	Subscribe(*SubscribeRequest, SeaweedMessaging_SubscribeServer) error
+	Publish(SeaweedMessaging_PublishServer) error
+	ConfigureTopic(context.Context, *ConfigureTopicRequest) (*ConfigureTopicResponse, error)
+	GetTopicConfiguration(context.Context, *GetTopicConfigurationRequest) (*GetTopicConfigurationResponse, error)
+}
+
+func RegisterSeaweedMessagingServer(s *grpc.Server, srv SeaweedMessagingServer) {
+	s.RegisterService(&_SeaweedMessaging_serviceDesc, srv)
+}
+
+func _SeaweedMessaging_Subscribe_Handler(srv interface{}, stream grpc.ServerStream) error {
+	m := new(SubscribeRequest)
+	if err := stream.RecvMsg(m); err != nil {
+		return err
+	}
+	return srv.(SeaweedMessagingServer).Subscribe(m, &seaweedMessagingSubscribeServer{stream})
+}
+
+type SeaweedMessaging_SubscribeServer interface {
+	Send(*Message) error
+	grpc.ServerStream
+}
+
+type seaweedMessagingSubscribeServer struct {
+	grpc.ServerStream
+}
+
+func (x *seaweedMessagingSubscribeServer) Send(m *Message) error {
+	return x.ServerStream.SendMsg(m)
+}
+
+func _SeaweedMessaging_Publish_Handler(srv interface{}, stream grpc.ServerStream) error {
+	return srv.(SeaweedMessagingServer).Publish(&seaweedMessagingPublishServer{stream})
+}
+
+type SeaweedMessaging_PublishServer interface {
+	Send(*PublishResponse) error
+	Recv() (*PublishRequest, error)
+	grpc.ServerStream
+}
+
+type seaweedMessagingPublishServer struct {
+	grpc.ServerStream
+}
+
+func (x *seaweedMessagingPublishServer) Send(m *PublishResponse) error {
+	return x.ServerStream.SendMsg(m)
+}
+
+func (x *seaweedMessagingPublishServer) Recv() (*PublishRequest, error) {
+	m := new(PublishRequest)
+	if err := x.ServerStream.RecvMsg(m); err != nil {
+		return nil, err
+	}
+	return m, nil
+}
+
+func _SeaweedMessaging_ConfigureTopic_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(ConfigureTopicRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(SeaweedMessagingServer).ConfigureTopic(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: "/messaging_pb.SeaweedMessaging/ConfigureTopic",
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(SeaweedMessagingServer).ConfigureTopic(ctx, req.(*ConfigureTopicRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+func _SeaweedMessaging_GetTopicConfiguration_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(GetTopicConfigurationRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(SeaweedMessagingServer).GetTopicConfiguration(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: "/messaging_pb.SeaweedMessaging/GetTopicConfiguration",
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(SeaweedMessagingServer).GetTopicConfiguration(ctx, req.(*GetTopicConfigurationRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+var _SeaweedMessaging_serviceDesc = grpc.ServiceDesc{
+	ServiceName: "messaging_pb.SeaweedMessaging",
+	HandlerType: (*SeaweedMessagingServer)(nil),
+	Methods: []grpc.MethodDesc{
+		{
+			MethodName: "ConfigureTopic",
+			Handler:    _SeaweedMessaging_ConfigureTopic_Handler,
+		},
+		{
+			MethodName: "GetTopicConfiguration",
+			Handler:    _SeaweedMessaging_GetTopicConfiguration_Handler,
+		},
+	},
+	Streams: []grpc.StreamDesc{
+		{
+			StreamName:    "Subscribe",
+			Handler:       _SeaweedMessaging_Subscribe_Handler,
+			ServerStreams: true,
+		},
+		{
+			StreamName:    "Publish",
+			Handler:       _SeaweedMessaging_Publish_Handler,
+			ServerStreams: true,
+			ClientStreams: true,
+		},
+	},
+	Metadata: "messaging.proto",
+}
+
+func init() { proto.RegisterFile("messaging.proto", fileDescriptor0) }
+
+var fileDescriptor0 = []byte{
+	// 586 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xb4, 0x95, 0xdb, 0x8b, 0xda, 0x4e,
+	0x14, 0xc7, 0x9d, 0xc4, 0xcb, 0x2f, 0x67, 0xbd, 0x31, 0xfc, 0x2c, 0x41, 0x5c, 0x09, 0x69, 0xa1,
+	0xe9, 0x85, 0x20, 0xf6, 0x65, 0x91, 0x42, 0x51, 0xb1, 0xed, 0x82, 0x16, 0x19, 0x7d, 0x2d, 0x4b,
+	0xcc, 0xce, 0xba, 0xa1, 0x9a, 0xa4, 0x99, 0x49, 0xcb, 0xfe, 0x0d, 0x7d, 0xdd, 0x7f, 0xab, 0x7f,
+	0x51, 0x5f, 0x4a, 0xae, 0x26, 0x92, 0x95, 0xd2, 0xcb, 0x5b, 0xe6, 0x3b, 0x67, 0xce, 0xf7, 0x9c,
+	0xcf, 0x49, 0x26, 0xd0, 0xda, 0x53, 0xc6, 0x8c, 0xad, 0x65, 0x6f, 0x75, 0xd7, 0x73, 0xb8, 0x83,
+	0xeb, 0xa9, 0x70, 0xe5, 0x6e, 0xd4, 0x7b, 0x01, 0xda, 0x2b, 0x7f, 0xc3, 0x4c, 0xcf, 0xda, 0x50,
+	0x42, 0x3f, 0xfb, 0x94, 0x71, 0xdc, 0x03, 0xc9, 0x36, 0xf6, 0x94, 0xb9, 0x86, 0x49, 0x65, 0xa4,
+	0x20, 0x4d, 0x22, 0x07, 0x01, 0xff, 0x0f, 0x15, 0xee, 0xb8, 0x96, 0x29, 0x0b, 0xe1, 0x4e, 0xb4,
+	0x08, 0xce, 0xb8, 0x86, 0xc7, 0x2d, 0x6e, 0x39, 0xb6, 0x2c, 0x2a, 0x48, 0xab, 0x90, 0x83, 0x80,
+	0x09, 0x34, 0x18, 0x37, 0x3c, 0xbe, 0x74, 0x58, 0x14, 0x51, 0x56, 0x90, 0xd6, 0x1c, 0xbe, 0xd4,
+	0xb3, 0xc5, 0xe8, 0xc7, 0x85, 0xe8, 0xab, 0xec, 0x19, 0x92, 0x4f, 0x81, 0x15, 0x38, 0xe3, 0xd6,
+	0x9e, 0x32, 0x6e, 0xec, 0xdd, 0x0f, 0x4c, 0xae, 0x28, 0x48, 0x13, 0x49, 0x56, 0x52, 0x2f, 0xa0,
+	0x91, 0xcb, 0x80, 0x01, 0xaa, 0xf3, 0xf1, 0x7a, 0xb6, 0x5a, 0xb7, 0x4b, 0xb8, 0x0e, 0xff, 0xcd,
+	0xc6, 0x64, 0x7e, 0x19, 0xac, 0x10, 0x6e, 0x80, 0xb4, 0xbe, 0x5c, 0xcc, 0x56, 0xeb, 0xf1, 0x62,
+	0xd9, 0x16, 0xd4, 0xef, 0x08, 0x6a, 0x8b, 0xb0, 0x34, 0x8a, 0x15, 0x90, 0xd2, 0xa4, 0x21, 0x0d,
+	0x71, 0x22, 0x0c, 0x10, 0x39, 0x88, 0xb8, 0x0d, 0xe2, 0x27, 0x7a, 0x17, 0xf2, 0xa8, 0x93, 0xe0,
+	0x31, 0x60, 0xf4, 0xc5, 0xd8, 0xf9, 0x34, 0x24, 0x51, 0x27, 0xd1, 0x02, 0xbf, 0x86, 0xda, 0x2d,
+	0x35, 0xae, 0xa9, 0xc7, 0xe4, 0xb2, 0x22, 0x6a, 0x67, 0x43, 0x35, 0xdf, 0x7f, 0xec, 0xa8, 0xbf,
+	0x8f, 0x82, 0x66, 0x36, 0xf7, 0xee, 0x48, 0x72, 0xa4, 0x3b, 0x82, 0x7a, 0x76, 0x23, 0x71, 0x8d,
+	0xe6, 0x93, 0x77, 0x15, 0x32, 0xae, 0x23, 0xe1, 0x02, 0xa9, 0xdf, 0x04, 0x68, 0x2e, 0xfd, 0xcd,
+	0xce, 0x62, 0xb7, 0x7f, 0x6d, 0xc8, 0xe5, 0xe3, 0x21, 0xc7, 0x05, 0x55, 0x0a, 0x30, 0x54, 0xb3,
+	0x18, 0xa6, 0x07, 0x0c, 0xb5, 0x10, 0xc3, 0xb3, 0x3c, 0x86, 0x7c, 0xa1, 0xff, 0x80, 0xc6, 0x08,
+	0x5a, 0xa9, 0x07, 0x73, 0x1d, 0x9b, 0x51, 0xfc, 0x14, 0x5a, 0x69, 0x23, 0x57, 0xa6, 0xe3, 0xdb,
+	0x3c, 0x4c, 0x55, 0x21, 0xcd, 0x54, 0x9e, 0x06, 0xaa, 0x7a, 0x8f, 0xa0, 0x33, 0x75, 0xec, 0x1b,
+	0x6b, 0xeb, 0x7b, 0x74, 0x1d, 0x50, 0xf9, 0x13, 0xa0, 0x05, 0xb6, 0x62, 0x91, 0x2d, 0xee, 0x03,
+	0x98, 0xce, 0x6e, 0x47, 0xcd, 0x14, 0xbd, 0x44, 0x32, 0x8a, 0x2a, 0xc3, 0xa3, 0xe3, 0xaa, 0xa2,
+	0xce, 0x54, 0x02, 0xbd, 0x77, 0x94, 0x87, 0x5a, 0x12, 0x61, 0x84, 0x9f, 0xd3, 0xef, 0x97, 0xad,
+	0xbe, 0x81, 0xf3, 0x07, 0x72, 0xc6, 0x38, 0xfb, 0x00, 0x69, 0x03, 0x2c, 0x6e, 0x29, 0xa3, 0x0c,
+	0x7f, 0x04, 0xd7, 0x0e, 0x35, 0xbe, 0x52, 0x7a, 0xbd, 0x48, 0x46, 0x8f, 0xdf, 0x82, 0x94, 0xde,
+	0x00, 0xb8, 0x7f, 0xfa, 0x6a, 0xe8, 0x76, 0x0a, 0x3f, 0x1d, 0xb5, 0x34, 0x40, 0x78, 0x0e, 0xb5,
+	0x78, 0xbc, 0xb8, 0x77, 0xea, 0xcd, 0xea, 0x9e, 0x3f, 0xb0, 0x1b, 0x93, 0x2b, 0x69, 0x68, 0x80,
+	0xf0, 0x47, 0x68, 0xe6, 0xc9, 0xe2, 0xc7, 0xf9, 0x63, 0x85, 0x6f, 0x43, 0xf7, 0xc9, 0xe9, 0xa0,
+	0xc4, 0x02, 0x7b, 0xd0, 0x29, 0x44, 0x89, 0x9f, 0xe7, 0x13, 0x9c, 0x9a, 0x61, 0xf7, 0xc5, 0x2f,
+	0xc5, 0x26, 0x9e, 0x13, 0x15, 0xda, 0x2c, 0x82, 0x7f, 0xc3, 0x74, 0x73, 0x67, 0x51, 0x9b, 0x4f,
+	0x9a, 0xe9, 0x1c, 0x96, 0xc1, 0x6f, 0x62, 0x53, 0x0d, 0xff, 0x16, 0xaf, 0x7e, 0x06, 0x00, 0x00,
+	0xff, 0xff, 0x83, 0x4f, 0x5b, 0x91, 0x40, 0x06, 0x00, 0x00,
+}
diff --git a/weed/pb/queue.proto b/weed/pb/queue.proto
deleted file mode 100644
index 39b6ee05a..000000000
--- a/weed/pb/queue.proto
+++ /dev/null
@@ -1,66 +0,0 @@
-syntax = "proto3";
-
-package queue_pb;
-
-option java_package = "seaweedfs.client";
-option java_outer_classname = "QueueProto";
-
-//////////////////////////////////////////////////
-
-service SeaweedQueue {
-
-    rpc StreamWrite (stream WriteMessageRequest) returns (stream WriteMessageResponse) {
-    }
-
-    rpc StreamRead (ReadMessageRequest) returns (stream ReadMessageResponse) {
-    }
-
-    rpc ConfigureTopic (ConfigureTopicRequest) returns (ConfigureTopicResponse) {
-    }
-
-    rpc DeleteTopic (DeleteTopicRequest) returns (DeleteTopicResponse) {
-    }
-
-}
-
-//////////////////////////////////////////////////
-
-
-message WriteMessageRequest {
-    string topic = 1;
-    int64 event_ns = 2;
-    bytes partition_key = 3;
-    bytes data = 4;
-}
-
-message WriteMessageResponse {
-    string error = 1;
-    int64 ack_ns = 2;
-}
-
-message ReadMessageRequest {
-    string topic = 1;
-    int64 start_ns = 2;
-}
-
-message ReadMessageResponse {
-    string error = 1;
-    int64 event_ns = 2;
-    bytes data = 3;
-}
-
-message ConfigureTopicRequest {
-    string topic = 1;
-    int64 ttl_seconds = 2;
-    int32 partition_count = 3;
-}
-message ConfigureTopicResponse {
-    string error = 1;
-}
-
-message DeleteTopicRequest {
-    string topic = 1;
-}
-message DeleteTopicResponse {
-    string error = 1;
-}
diff --git a/weed/pb/queue_pb/queue.pb.go b/weed/pb/queue_pb/queue.pb.go
deleted file mode 100644
index 8ec4d62aa..000000000
--- a/weed/pb/queue_pb/queue.pb.go
+++ /dev/null
@@ -1,516 +0,0 @@
-// Code generated by protoc-gen-go.
-// source: queue.proto
-// DO NOT EDIT!
-
-/*
-Package queue_pb is a generated protocol buffer package.
-
-It is generated from these files:
-	queue.proto
-
-It has these top-level messages:
-	WriteMessageRequest
-	WriteMessageResponse
-	ReadMessageRequest
-	ReadMessageResponse
-	ConfigureTopicRequest
-	ConfigureTopicResponse
-	DeleteTopicRequest
-	DeleteTopicResponse
-*/
-package queue_pb
-
-import proto "github.com/golang/protobuf/proto"
-import fmt "fmt"
-import math "math"
-
-import (
-	context "golang.org/x/net/context"
-	grpc "google.golang.org/grpc"
-)
-
-// Reference imports to suppress errors if they are not otherwise used.
-var _ = proto.Marshal
-var _ = fmt.Errorf
-var _ = math.Inf
-
-// This is a compile-time assertion to ensure that this generated file
-// is compatible with the proto package it is being compiled against.
-// A compilation error at this line likely means your copy of the
-// proto package needs to be updated.
-const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package
-
-type WriteMessageRequest struct {
-	Topic        string `protobuf:"bytes,1,opt,name=topic" json:"topic,omitempty"`
-	EventNs      int64  `protobuf:"varint,2,opt,name=event_ns,json=eventNs" json:"event_ns,omitempty"`
-	PartitionKey []byte `protobuf:"bytes,3,opt,name=partition_key,json=partitionKey,proto3" json:"partition_key,omitempty"`
-	Data         []byte `protobuf:"bytes,4,opt,name=data,proto3" json:"data,omitempty"`
-}
-
-func (m *WriteMessageRequest) Reset()                    { *m = WriteMessageRequest{} }
-func (m *WriteMessageRequest) String() string            { return proto.CompactTextString(m) }
-func (*WriteMessageRequest) ProtoMessage()               {}
-func (*WriteMessageRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{0} }
-
-func (m *WriteMessageRequest) GetTopic() string {
-	if m != nil {
-		return m.Topic
-	}
-	return ""
-}
-
-func (m *WriteMessageRequest) GetEventNs() int64 {
-	if m != nil {
-		return m.EventNs
-	}
-	return 0
-}
-
-func (m *WriteMessageRequest) GetPartitionKey() []byte {
-	if m != nil {
-		return m.PartitionKey
-	}
-	return nil
-}
-
-func (m *WriteMessageRequest) GetData() []byte {
-	if m != nil {
-		return m.Data
-	}
-	return nil
-}
-
-type WriteMessageResponse struct {
-	Error string `protobuf:"bytes,1,opt,name=error" json:"error,omitempty"`
-	AckNs int64  `protobuf:"varint,2,opt,name=ack_ns,json=ackNs" json:"ack_ns,omitempty"`
-}
-
-func (m *WriteMessageResponse) Reset()                    { *m = WriteMessageResponse{} }
-func (m *WriteMessageResponse) String() string            { return proto.CompactTextString(m) }
-func (*WriteMessageResponse) ProtoMessage()               {}
-func (*WriteMessageResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{1} }
-
-func (m *WriteMessageResponse) GetError() string {
-	if m != nil {
-		return m.Error
-	}
-	return ""
-}
-
-func (m *WriteMessageResponse) GetAckNs() int64 {
-	if m != nil {
-		return m.AckNs
-	}
-	return 0
-}
-
-type ReadMessageRequest struct {
-	Topic   string `protobuf:"bytes,1,opt,name=topic" json:"topic,omitempty"`
-	StartNs int64  `protobuf:"varint,2,opt,name=start_ns,json=startNs" json:"start_ns,omitempty"`
-}
-
-func (m *ReadMessageRequest) Reset()                    { *m = ReadMessageRequest{} }
-func (m *ReadMessageRequest) String() string            { return proto.CompactTextString(m) }
-func (*ReadMessageRequest) ProtoMessage()               {}
-func (*ReadMessageRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{2} }
-
-func (m *ReadMessageRequest) GetTopic() string {
-	if m != nil {
-		return m.Topic
-	}
-	return ""
-}
-
-func (m *ReadMessageRequest) GetStartNs() int64 {
-	if m != nil {
-		return m.StartNs
-	}
-	return 0
-}
-
-type ReadMessageResponse struct {
-	Error   string `protobuf:"bytes,1,opt,name=error" json:"error,omitempty"`
-	EventNs int64  `protobuf:"varint,2,opt,name=event_ns,json=eventNs" json:"event_ns,omitempty"`
-	Data    []byte `protobuf:"bytes,3,opt,name=data,proto3" json:"data,omitempty"`
-}
-
-func (m *ReadMessageResponse) Reset()                    { *m = ReadMessageResponse{} }
-func (m *ReadMessageResponse) String() string            { return proto.CompactTextString(m) }
-func (*ReadMessageResponse) ProtoMessage()               {}
-func (*ReadMessageResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{3} }
-
-func (m *ReadMessageResponse) GetError() string {
-	if m != nil {
-		return m.Error
-	}
-	return ""
-}
-
-func (m *ReadMessageResponse) GetEventNs() int64 {
-	if m != nil {
-		return m.EventNs
-	}
-	return 0
-}
-
-func (m *ReadMessageResponse) GetData() []byte {
-	if m != nil {
-		return m.Data
-	}
-	return nil
-}
-
-type ConfigureTopicRequest struct {
-	Topic          string `protobuf:"bytes,1,opt,name=topic" json:"topic,omitempty"`
-	TtlSeconds     int64  `protobuf:"varint,2,opt,name=ttl_seconds,json=ttlSeconds" json:"ttl_seconds,omitempty"`
-	PartitionCount int32  `protobuf:"varint,3,opt,name=partition_count,json=partitionCount" json:"partition_count,omitempty"`
-}
-
-func (m *ConfigureTopicRequest) Reset()                    { *m = ConfigureTopicRequest{} }
-func (m *ConfigureTopicRequest) String() string            { return proto.CompactTextString(m) }
-func (*ConfigureTopicRequest) ProtoMessage()               {}
-func (*ConfigureTopicRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{4} }
-
-func (m *ConfigureTopicRequest) GetTopic() string {
-	if m != nil {
-		return m.Topic
-	}
-	return ""
-}
-
-func (m *ConfigureTopicRequest) GetTtlSeconds() int64 {
-	if m != nil {
-		return m.TtlSeconds
-	}
-	return 0
-}
-
-func (m *ConfigureTopicRequest) GetPartitionCount() int32 {
-	if m != nil {
-		return m.PartitionCount
-	}
-	return 0
-}
-
-type ConfigureTopicResponse struct {
-	Error string `protobuf:"bytes,1,opt,name=error" json:"error,omitempty"`
-}
-
-func (m *ConfigureTopicResponse) Reset()                    { *m = ConfigureTopicResponse{} }
-func (m *ConfigureTopicResponse) String() string            { return proto.CompactTextString(m) }
-func (*ConfigureTopicResponse) ProtoMessage()               {}
-func (*ConfigureTopicResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{5} }
-
-func (m *ConfigureTopicResponse) GetError() string {
-	if m != nil {
-		return m.Error
-	}
-	return ""
-}
-
-type DeleteTopicRequest struct {
-	Topic string `protobuf:"bytes,1,opt,name=topic" json:"topic,omitempty"`
-}
-
-func (m *DeleteTopicRequest) Reset()                    { *m = DeleteTopicRequest{} }
-func (m *DeleteTopicRequest) String() string            { return proto.CompactTextString(m) }
-func (*DeleteTopicRequest) ProtoMessage()               {}
-func (*DeleteTopicRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{6} }
-
-func (m *DeleteTopicRequest) GetTopic() string {
-	if m != nil {
-		return m.Topic
-	}
-	return ""
-}
-
-type DeleteTopicResponse struct {
-	Error string `protobuf:"bytes,1,opt,name=error" json:"error,omitempty"`
-}
-
-func (m *DeleteTopicResponse) Reset()                    { *m = DeleteTopicResponse{} }
-func (m *DeleteTopicResponse) String() string            { return proto.CompactTextString(m) }
-func (*DeleteTopicResponse) ProtoMessage()               {}
-func (*DeleteTopicResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{7} }
-
-func (m *DeleteTopicResponse) GetError() string {
-	if m != nil {
-		return m.Error
-	}
-	return ""
-}
-
-func init() {
-	proto.RegisterType((*WriteMessageRequest)(nil), "queue_pb.WriteMessageRequest")
-	proto.RegisterType((*WriteMessageResponse)(nil), "queue_pb.WriteMessageResponse")
-	proto.RegisterType((*ReadMessageRequest)(nil), "queue_pb.ReadMessageRequest")
-	proto.RegisterType((*ReadMessageResponse)(nil), "queue_pb.ReadMessageResponse")
-	proto.RegisterType((*ConfigureTopicRequest)(nil), "queue_pb.ConfigureTopicRequest")
-	proto.RegisterType((*ConfigureTopicResponse)(nil), "queue_pb.ConfigureTopicResponse")
-	proto.RegisterType((*DeleteTopicRequest)(nil), "queue_pb.DeleteTopicRequest")
-	proto.RegisterType((*DeleteTopicResponse)(nil), "queue_pb.DeleteTopicResponse")
-}
-
-// Reference imports to suppress errors if they are not otherwise used.
-var _ context.Context
-var _ grpc.ClientConn
-
-// This is a compile-time assertion to ensure that this generated file
-// is compatible with the grpc package it is being compiled against.
-const _ = grpc.SupportPackageIsVersion4
-
-// Client API for SeaweedQueue service
-
-type SeaweedQueueClient interface {
-	StreamWrite(ctx context.Context, opts ...grpc.CallOption) (SeaweedQueue_StreamWriteClient, error)
-	StreamRead(ctx context.Context, in *ReadMessageRequest, opts ...grpc.CallOption) (SeaweedQueue_StreamReadClient, error)
-	ConfigureTopic(ctx context.Context, in *ConfigureTopicRequest, opts ...grpc.CallOption) (*ConfigureTopicResponse, error)
-	DeleteTopic(ctx context.Context, in *DeleteTopicRequest, opts ...grpc.CallOption) (*DeleteTopicResponse, error)
-}
-
-type seaweedQueueClient struct {
-	cc *grpc.ClientConn
-}
-
-func NewSeaweedQueueClient(cc *grpc.ClientConn) SeaweedQueueClient {
-	return &seaweedQueueClient{cc}
-}
-
-func (c *seaweedQueueClient) StreamWrite(ctx context.Context, opts ...grpc.CallOption) (SeaweedQueue_StreamWriteClient, error) {
-	stream, err := grpc.NewClientStream(ctx, &_SeaweedQueue_serviceDesc.Streams[0], c.cc, "/queue_pb.SeaweedQueue/StreamWrite", opts...)
-	if err != nil {
-		return nil, err
-	}
-	x := &seaweedQueueStreamWriteClient{stream}
-	return x, nil
-}
-
-type SeaweedQueue_StreamWriteClient interface {
-	Send(*WriteMessageRequest) error
-	Recv() (*WriteMessageResponse, error)
-	grpc.ClientStream
-}
-
-type seaweedQueueStreamWriteClient struct {
-	grpc.ClientStream
-}
-
-func (x *seaweedQueueStreamWriteClient) Send(m *WriteMessageRequest) error {
-	return x.ClientStream.SendMsg(m)
-}
-
-func (x *seaweedQueueStreamWriteClient) Recv() (*WriteMessageResponse, error) {
-	m := new(WriteMessageResponse)
-	if err := x.ClientStream.RecvMsg(m); err != nil {
-		return nil, err
-	}
-	return m, nil
-}
-
-func (c *seaweedQueueClient) StreamRead(ctx context.Context, in *ReadMessageRequest, opts ...grpc.CallOption) (SeaweedQueue_StreamReadClient, error) {
-	stream, err := grpc.NewClientStream(ctx, &_SeaweedQueue_serviceDesc.Streams[1], c.cc, "/queue_pb.SeaweedQueue/StreamRead", opts...)
-	if err != nil {
-		return nil, err
-	}
-	x := &seaweedQueueStreamReadClient{stream}
-	if err := x.ClientStream.SendMsg(in); err != nil {
-		return nil, err
-	}
-	if err := x.ClientStream.CloseSend(); err != nil {
-		return nil, err
-	}
-	return x, nil
-}
-
-type SeaweedQueue_StreamReadClient interface {
-	Recv() (*ReadMessageResponse, error)
-	grpc.ClientStream
-}
-
-type seaweedQueueStreamReadClient struct {
-	grpc.ClientStream
-}
-
-func (x *seaweedQueueStreamReadClient) Recv() (*ReadMessageResponse, error) {
-	m := new(ReadMessageResponse)
-	if err := x.ClientStream.RecvMsg(m); err != nil {
-		return nil, err
-	}
-	return m, nil
-}
-
-func (c *seaweedQueueClient) ConfigureTopic(ctx context.Context, in *ConfigureTopicRequest, opts ...grpc.CallOption) (*ConfigureTopicResponse, error) {
-	out := new(ConfigureTopicResponse)
-	err := grpc.Invoke(ctx, "/queue_pb.SeaweedQueue/ConfigureTopic", in, out, c.cc, opts...)
-	if err != nil {
-		return nil, err
-	}
-	return out, nil
-}
-
-func (c *seaweedQueueClient) DeleteTopic(ctx context.Context, in *DeleteTopicRequest, opts ...grpc.CallOption) (*DeleteTopicResponse, error) {
-	out := new(DeleteTopicResponse)
-	err := grpc.Invoke(ctx, "/queue_pb.SeaweedQueue/DeleteTopic", in, out, c.cc, opts...)
-	if err != nil {
-		return nil, err
-	}
-	return out, nil
-}
-
-// Server API for SeaweedQueue service
-
-type SeaweedQueueServer interface {
-	StreamWrite(SeaweedQueue_StreamWriteServer) error
-	StreamRead(*ReadMessageRequest, SeaweedQueue_StreamReadServer) error
-	ConfigureTopic(context.Context, *ConfigureTopicRequest) (*ConfigureTopicResponse, error)
-	DeleteTopic(context.Context, *DeleteTopicRequest) (*DeleteTopicResponse, error)
-}
-
-func RegisterSeaweedQueueServer(s *grpc.Server, srv SeaweedQueueServer) {
-	s.RegisterService(&_SeaweedQueue_serviceDesc, srv)
-}
-
-func _SeaweedQueue_StreamWrite_Handler(srv interface{}, stream grpc.ServerStream) error {
-	return srv.(SeaweedQueueServer).StreamWrite(&seaweedQueueStreamWriteServer{stream})
-}
-
-type SeaweedQueue_StreamWriteServer interface {
-	Send(*WriteMessageResponse) error
-	Recv() (*WriteMessageRequest, error)
-	grpc.ServerStream
-}
-
-type seaweedQueueStreamWriteServer struct {
-	grpc.ServerStream
-}
-
-func (x *seaweedQueueStreamWriteServer) Send(m *WriteMessageResponse) error {
-	return x.ServerStream.SendMsg(m)
-}
-
-func (x *seaweedQueueStreamWriteServer) Recv() (*WriteMessageRequest, error) {
-	m := new(WriteMessageRequest)
-	if err := x.ServerStream.RecvMsg(m); err != nil {
-		return nil, err
-	}
-	return m, nil
-}
-
-func _SeaweedQueue_StreamRead_Handler(srv interface{}, stream grpc.ServerStream) error {
-	m := new(ReadMessageRequest)
-	if err := stream.RecvMsg(m); err != nil {
-		return err
-	}
-	return srv.(SeaweedQueueServer).StreamRead(m, &seaweedQueueStreamReadServer{stream})
-}
-
-type SeaweedQueue_StreamReadServer interface {
-	Send(*ReadMessageResponse) error
-	grpc.ServerStream
-}
-
-type seaweedQueueStreamReadServer struct {
-	grpc.ServerStream
-}
-
-func (x *seaweedQueueStreamReadServer) Send(m *ReadMessageResponse) error {
-	return x.ServerStream.SendMsg(m)
-}
-
-func _SeaweedQueue_ConfigureTopic_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
-	in := new(ConfigureTopicRequest)
-	if err := dec(in); err != nil {
-		return nil, err
-	}
-	if interceptor == nil {
-		return srv.(SeaweedQueueServer).ConfigureTopic(ctx, in)
-	}
-	info := &grpc.UnaryServerInfo{
-		Server:     srv,
-		FullMethod: "/queue_pb.SeaweedQueue/ConfigureTopic",
-	}
-	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
-		return srv.(SeaweedQueueServer).ConfigureTopic(ctx, req.(*ConfigureTopicRequest))
-	}
-	return interceptor(ctx, in, info, handler)
-}
-
-func _SeaweedQueue_DeleteTopic_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
-	in := new(DeleteTopicRequest)
-	if err := dec(in); err != nil {
-		return nil, err
-	}
-	if interceptor == nil {
-		return srv.(SeaweedQueueServer).DeleteTopic(ctx, in)
-	}
-	info := &grpc.UnaryServerInfo{
-		Server:     srv,
-		FullMethod: "/queue_pb.SeaweedQueue/DeleteTopic",
-	}
-	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
-		return srv.(SeaweedQueueServer).DeleteTopic(ctx, req.(*DeleteTopicRequest))
-	}
-	return interceptor(ctx, in, info, handler)
-}
-
-var _SeaweedQueue_serviceDesc = grpc.ServiceDesc{
-	ServiceName: "queue_pb.SeaweedQueue",
-	HandlerType: (*SeaweedQueueServer)(nil),
-	Methods: []grpc.MethodDesc{
-		{
-			MethodName: "ConfigureTopic",
-			Handler:    _SeaweedQueue_ConfigureTopic_Handler,
-		},
-		{
-			MethodName: "DeleteTopic",
-			Handler:    _SeaweedQueue_DeleteTopic_Handler,
-		},
-	},
-	Streams: []grpc.StreamDesc{
-		{
-			StreamName:    "StreamWrite",
-			Handler:       _SeaweedQueue_StreamWrite_Handler,
-			ServerStreams: true,
-			ClientStreams: true,
-		},
-		{
-			StreamName:    "StreamRead",
-			Handler:       _SeaweedQueue_StreamRead_Handler,
-			ServerStreams: true,
-		},
-	},
-	Metadata: "queue.proto",
-}
-
-func init() { proto.RegisterFile("queue.proto", fileDescriptor0) }
-
-var fileDescriptor0 = []byte{
-	// 429 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0x8c, 0x53, 0xcd, 0x6e, 0xd3, 0x40,
-	0x10, 0xae, 0x9b, 0xa6, 0x94, 0x49, 0x28, 0x68, 0xd2, 0xa2, 0x10, 0xd1, 0x36, 0x5a, 0x0e, 0x44,
-	0x20, 0x59, 0x15, 0xbc, 0x41, 0x03, 0x27, 0x68, 0x04, 0x0e, 0x08, 0x89, 0x8b, 0xb5, 0xb5, 0xa7,
-	0x95, 0x15, 0xb3, 0xeb, 0xee, 0x8e, 0xa9, 0x7a, 0xe2, 0x2d, 0x79, 0x1e, 0xe4, 0xb5, 0x5c, 0xdb,
-	0x34, 0xb1, 0x7a, 0xf3, 0xcc, 0x78, 0xe7, 0xfb, 0xd9, 0x6f, 0x61, 0x70, 0x9d, 0x53, 0x4e, 0x7e,
-	0x66, 0x34, 0x6b, 0xdc, 0x73, 0x45, 0x98, 0x5d, 0x88, 0x3f, 0x30, 0xfa, 0x61, 0x12, 0xa6, 0x73,
-	0xb2, 0x56, 0x5e, 0x51, 0x40, 0xd7, 0x39, 0x59, 0xc6, 0x03, 0xe8, 0xb3, 0xce, 0x92, 0x68, 0xec,
-	0x4d, 0xbd, 0xd9, 0xe3, 0xa0, 0x2c, 0xf0, 0x05, 0xec, 0xd1, 0x6f, 0x52, 0x1c, 0x2a, 0x3b, 0xde,
-	0x9e, 0x7a, 0xb3, 0x5e, 0xf0, 0xc8, 0xd5, 0x0b, 0x8b, 0xaf, 0xe0, 0x49, 0x26, 0x0d, 0x27, 0x9c,
-	0x68, 0x15, 0xae, 0xe8, 0x76, 0xdc, 0x9b, 0x7a, 0xb3, 0x61, 0x30, 0xbc, 0x6b, 0x7e, 0xa2, 0x5b,
-	0x44, 0xd8, 0x89, 0x25, 0xcb, 0xf1, 0x8e, 0x9b, 0xb9, 0x6f, 0x31, 0x87, 0x83, 0x36, 0x01, 0x9b,
-	0x69, 0x65, 0xa9, 0x60, 0x40, 0xc6, 0x68, 0x53, 0x31, 0x70, 0x05, 0x1e, 0xc2, 0xae, 0x8c, 0x56,
-	0x35, 0x7e, 0x5f, 0x46, 0xab, 0x85, 0x15, 0x1f, 0x01, 0x03, 0x92, 0xf1, 0x43, 0x45, 0x58, 0x96,
-	0xa6, 0x29, 0xc2, 0xd5, 0x0b, 0x2b, 0x7e, 0xc2, 0xa8, 0xb5, 0xa6, 0x93, 0x4a, 0x87, 0x19, 0x95,
-	0xce, 0x5e, 0x43, 0xe7, 0x0d, 0x1c, 0xce, 0xb5, 0xba, 0x4c, 0xae, 0x72, 0x43, 0xdf, 0x0a, 0x22,
-	0xdd, 0x2c, 0x4f, 0x60, 0xc0, 0x9c, 0x86, 0x96, 0x22, 0xad, 0xe2, 0x0a, 0x00, 0x98, 0xd3, 0x65,
-	0xd9, 0xc1, 0xd7, 0xf0, 0xb4, 0x36, 0x3c, 0xd2, 0xb9, 0x62, 0x07, 0xd7, 0x0f, 0xf6, 0xef, 0xda,
-	0xf3, 0xa2, 0x2b, 0x7c, 0x78, 0xfe, 0x3f, 0x70, 0x97, 0x2e, 0xf1, 0x06, 0xf0, 0x03, 0xa5, 0xc4,
-	0x0f, 0x60, 0x29, 0xde, 0xc2, 0xa8, 0xf5, 0x6f, 0xd7, 0xe2, 0x77, 0x7f, 0xb7, 0x61, 0xb8, 0x24,
-	0x79, 0x43, 0x14, 0x7f, 0x2d, 0xe2, 0x87, 0x01, 0x0c, 0x96, 0x6c, 0x48, 0xfe, 0x72, 0x01, 0xc0,
-	0x23, 0xbf, 0x4a, 0xa5, 0xbf, 0x26, 0x92, 0x93, 0xe3, 0x4d, 0xe3, 0x12, 0x54, 0x6c, 0xcd, 0xbc,
-	0x53, 0x0f, 0xcf, 0x01, 0xca, 0x9d, 0xc5, 0x45, 0xe2, 0xcb, 0xfa, 0xcc, 0xfd, 0x7c, 0x4c, 0x8e,
-	0x36, 0x4c, 0xab, 0x85, 0xa7, 0x1e, 0x7e, 0x87, 0xfd, 0xb6, 0x79, 0x78, 0x52, 0x1f, 0x5a, 0x7b,
-	0x9f, 0x93, 0xe9, 0xe6, 0x1f, 0xaa, 0xc5, 0xf8, 0x19, 0x06, 0x0d, 0xdf, 0x9a, 0x34, 0xef, 0x5b,
-	0xdf, 0xa4, 0xb9, 0xc6, 0x6c, 0xb1, 0x75, 0x76, 0x0c, 0xcf, 0x6c, 0xe9, 0xeb, 0xa5, 0xf5, 0xa3,
-	0x34, 0x21, 0xc5, 0x67, 0xe0, 0x2c, 0xfe, 0x52, 0xbc, 0xf6, 0x8b, 0x5d, 0xf7, 0xe8, 0xdf, 0xff,
-	0x0b, 0x00, 0x00, 0xff, 0xff, 0x7d, 0x3e, 0x14, 0xd8, 0x03, 0x04, 0x00, 0x00,
-}
diff --git a/weed/server/msg_broker_grpc_server.go b/weed/server/msg_broker_grpc_server.go
index 8b13aac76..6feaa0c63 100644
--- a/weed/server/msg_broker_grpc_server.go
+++ b/weed/server/msg_broker_grpc_server.go
@@ -3,21 +3,21 @@ package weed_server
 import (
 	"context"
 
-	"github.com/chrislusf/seaweedfs/weed/pb/queue_pb"
+	"github.com/chrislusf/seaweedfs/weed/pb/messaging_pb"
 )
 
-func (broker *MessageBroker) ConfigureTopic(context.Context, *queue_pb.ConfigureTopicRequest) (*queue_pb.ConfigureTopicResponse, error) {
+func (broker *MessageBroker) Subscribe(request *messaging_pb.SubscribeRequest, server messaging_pb.SeaweedMessaging_SubscribeServer) error {
 	panic("implement me")
 }
 
-func (broker *MessageBroker) DeleteTopic(context.Context, *queue_pb.DeleteTopicRequest) (*queue_pb.DeleteTopicResponse, error) {
+func (broker *MessageBroker) Publish(server messaging_pb.SeaweedMessaging_PublishServer) error {
 	panic("implement me")
 }
 
-func (broker *MessageBroker) StreamWrite(queue_pb.SeaweedQueue_StreamWriteServer) error {
+func (broker *MessageBroker) ConfigureTopic(c context.Context, request *messaging_pb.ConfigureTopicRequest) (*messaging_pb.ConfigureTopicResponse, error) {
 	panic("implement me")
 }
 
-func (broker *MessageBroker) StreamRead(*queue_pb.ReadMessageRequest, queue_pb.SeaweedQueue_StreamReadServer) error {
+func (broker *MessageBroker) GetTopicConfiguration(c context.Context, request *messaging_pb.GetTopicConfigurationRequest) (*messaging_pb.GetTopicConfigurationResponse, error) {
 	panic("implement me")
 }

From f5a748d33c52a2874dbde92746a03140ef379296 Mon Sep 17 00:00:00 2001
From: Chris Lu <chris.lu@gmail.com>
Date: Thu, 16 Apr 2020 02:55:09 -0700
Subject: [PATCH 69/81] refactoring

---
 weed/command/msg_broker.go                    |  7 +-
 weed/messaging/msg_broker_grpc_server.go      | 98 +++++++++++++++++++
 .../msg_broker_server.go                      |  2 +-
 weed/server/msg_broker_grpc_server.go         | 23 -----
 weed/util/log_buffer/log_buffer.go            |  4 +-
 5 files changed, 104 insertions(+), 30 deletions(-)
 create mode 100644 weed/messaging/msg_broker_grpc_server.go
 rename weed/{server => messaging}/msg_broker_server.go (99%)
 delete mode 100644 weed/server/msg_broker_grpc_server.go

diff --git a/weed/command/msg_broker.go b/weed/command/msg_broker.go
index 3cb424298..f77582f03 100644
--- a/weed/command/msg_broker.go
+++ b/weed/command/msg_broker.go
@@ -8,13 +8,12 @@ import (
 
 	"google.golang.org/grpc/reflection"
 
+	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/messaging"
 	"github.com/chrislusf/seaweedfs/weed/pb"
 	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
 	"github.com/chrislusf/seaweedfs/weed/pb/messaging_pb"
 	"github.com/chrislusf/seaweedfs/weed/security"
-	weed_server "github.com/chrislusf/seaweedfs/weed/server"
-
-	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/util"
 )
 
@@ -79,7 +78,7 @@ func (msgBrokerOpt *QueueOptions) startQueueServer() bool {
 		}
 	}
 
-	qs, err := weed_server.NewMessageBroker(&weed_server.MessageBrokerOption{
+	qs, err := messaging.NewMessageBroker(&messaging.MessageBrokerOption{
 		Filers:             []string{*msgBrokerOpt.filer},
 		DefaultReplication: "",
 		MaxMB:              0,
diff --git a/weed/messaging/msg_broker_grpc_server.go b/weed/messaging/msg_broker_grpc_server.go
new file mode 100644
index 000000000..a29bc11b0
--- /dev/null
+++ b/weed/messaging/msg_broker_grpc_server.go
@@ -0,0 +1,98 @@
+package messaging
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"time"
+
+	"github.com/golang/protobuf/proto"
+
+	"github.com/chrislusf/seaweedfs/weed/filer2"
+	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/pb/messaging_pb"
+	"github.com/chrislusf/seaweedfs/weed/util/log_buffer"
+)
+
+func (broker *MessageBroker) Subscribe(request *messaging_pb.SubscribeRequest, server messaging_pb.SeaweedMessaging_SubscribeServer) error {
+	panic("implement me")
+}
+
+func (broker *MessageBroker) Publish(stream messaging_pb.SeaweedMessaging_PublishServer) error {
+
+	// process initial request
+	in, err := stream.Recv()
+	if err == io.EOF {
+		return nil
+	}
+	if err != nil {
+		return err
+	}
+	namespace, topic, partition := in.Namespace, in.Topic, in.Partition
+
+	updatesChan := make(chan int32)
+
+	go func() {
+		for update := range updatesChan {
+			if err := stream.Send(&messaging_pb.PublishResponse{
+				PartitionCount: update,
+			}); err != nil {
+				glog.V(0).Infof("err sending publish response: %v", err)
+				return
+			}
+		}
+	}()
+
+	logBuffer := log_buffer.NewLogBuffer(time.Minute, func(startTime, stopTime time.Time, buf []byte) {
+
+		//targetFile :=
+		fmt.Sprintf("%s/%s/%s/%04d-%02d-%02d/%02d-%02d.part%02d",
+			filer2.TopicsDir, namespace, topic,
+			startTime.Year(), startTime.Month(), startTime.Day(), startTime.Hour(), startTime.Minute(),
+			partition,
+		)
+
+		/*
+			if err := f.appendToFile(targetFile, buf); err != nil {
+				glog.V(0).Infof("log write failed %s: %v", targetFile, err)
+			}
+		*/
+
+	}, func() {
+		// notify subscribers
+	})
+
+	for {
+		in, err := stream.Recv()
+		if err == io.EOF {
+			return nil
+		}
+		if err != nil {
+			return err
+		}
+
+		m := &messaging_pb.Message{
+			Timestamp: time.Now().UnixNano(),
+			Key:       in.Key,
+			Value:     in.Value,
+			Headers:   in.Headers,
+		}
+
+		data, err := proto.Marshal(m)
+		if err != nil {
+			glog.Errorf("marshall error: %v\n", err)
+			continue
+		}
+
+		logBuffer.AddToBuffer(in.Key, data)
+
+	}
+}
+
+func (broker *MessageBroker) ConfigureTopic(c context.Context, request *messaging_pb.ConfigureTopicRequest) (*messaging_pb.ConfigureTopicResponse, error) {
+	panic("implement me")
+}
+
+func (broker *MessageBroker) GetTopicConfiguration(c context.Context, request *messaging_pb.GetTopicConfigurationRequest) (*messaging_pb.GetTopicConfigurationResponse, error) {
+	panic("implement me")
+}
diff --git a/weed/server/msg_broker_server.go b/weed/messaging/msg_broker_server.go
similarity index 99%
rename from weed/server/msg_broker_server.go
rename to weed/messaging/msg_broker_server.go
index a9d908581..9174ca4cf 100644
--- a/weed/server/msg_broker_server.go
+++ b/weed/messaging/msg_broker_server.go
@@ -1,4 +1,4 @@
-package weed_server
+package messaging
 
 import (
 	"context"
diff --git a/weed/server/msg_broker_grpc_server.go b/weed/server/msg_broker_grpc_server.go
deleted file mode 100644
index 6feaa0c63..000000000
--- a/weed/server/msg_broker_grpc_server.go
+++ /dev/null
@@ -1,23 +0,0 @@
-package weed_server
-
-import (
-	"context"
-
-	"github.com/chrislusf/seaweedfs/weed/pb/messaging_pb"
-)
-
-func (broker *MessageBroker) Subscribe(request *messaging_pb.SubscribeRequest, server messaging_pb.SeaweedMessaging_SubscribeServer) error {
-	panic("implement me")
-}
-
-func (broker *MessageBroker) Publish(server messaging_pb.SeaweedMessaging_PublishServer) error {
-	panic("implement me")
-}
-
-func (broker *MessageBroker) ConfigureTopic(c context.Context, request *messaging_pb.ConfigureTopicRequest) (*messaging_pb.ConfigureTopicResponse, error) {
-	panic("implement me")
-}
-
-func (broker *MessageBroker) GetTopicConfiguration(c context.Context, request *messaging_pb.GetTopicConfigurationRequest) (*messaging_pb.GetTopicConfigurationResponse, error) {
-	panic("implement me")
-}
diff --git a/weed/util/log_buffer/log_buffer.go b/weed/util/log_buffer/log_buffer.go
index 04fabdf8c..c7cb90549 100644
--- a/weed/util/log_buffer/log_buffer.go
+++ b/weed/util/log_buffer/log_buffer.go
@@ -51,7 +51,7 @@ func NewLogBuffer(flushInterval time.Duration, flushFn func(startTime, stopTime
 	return lb
 }
 
-func (m *LogBuffer) AddToBuffer(key, data []byte) {
+func (m *LogBuffer) AddToBuffer(partitionKey, data []byte) {
 
 	m.Lock()
 	defer func() {
@@ -65,7 +65,7 @@ func (m *LogBuffer) AddToBuffer(key, data []byte) {
 	ts := time.Now()
 	logEntry := &filer_pb.LogEntry{
 		TsNs:             ts.UnixNano(),
-		PartitionKeyHash: util.HashToInt32(key),
+		PartitionKeyHash: util.HashToInt32(partitionKey),
 		Data:             data,
 	}
 

From 508f3490a0a9a2eb0a44ebd45cb5ef03a931c7f9 Mon Sep 17 00:00:00 2001
From: Chris Lu <chris.lu@gmail.com>
Date: Thu, 16 Apr 2020 03:29:57 -0700
Subject: [PATCH 70/81] update messaging proto

---
 weed/messaging/msg_broker_grpc_server.go |  16 +-
 weed/pb/messaging.proto                  |  65 +++-
 weed/pb/messaging_pb/messaging.pb.go     | 408 +++++++++++++++++------
 3 files changed, 358 insertions(+), 131 deletions(-)

diff --git a/weed/messaging/msg_broker_grpc_server.go b/weed/messaging/msg_broker_grpc_server.go
index a29bc11b0..5b93b8f62 100644
--- a/weed/messaging/msg_broker_grpc_server.go
+++ b/weed/messaging/msg_broker_grpc_server.go
@@ -14,7 +14,7 @@ import (
 	"github.com/chrislusf/seaweedfs/weed/util/log_buffer"
 )
 
-func (broker *MessageBroker) Subscribe(request *messaging_pb.SubscribeRequest, server messaging_pb.SeaweedMessaging_SubscribeServer) error {
+func (broker *MessageBroker) Subscribe(server messaging_pb.SeaweedMessaging_SubscribeServer) error {
 	panic("implement me")
 }
 
@@ -28,14 +28,16 @@ func (broker *MessageBroker) Publish(stream messaging_pb.SeaweedMessaging_Publis
 	if err != nil {
 		return err
 	}
-	namespace, topic, partition := in.Namespace, in.Topic, in.Partition
+	namespace, topic, partition := in.Init.Namespace, in.Init.Topic, in.Init.Partition
 
 	updatesChan := make(chan int32)
 
 	go func() {
 		for update := range updatesChan {
 			if err := stream.Send(&messaging_pb.PublishResponse{
-				PartitionCount: update,
+				Config: &messaging_pb.PublishResponse_ConfigMessage{
+					PartitionCount: update,
+				},
 			}); err != nil {
 				glog.V(0).Infof("err sending publish response: %v", err)
 				return
@@ -73,9 +75,9 @@ func (broker *MessageBroker) Publish(stream messaging_pb.SeaweedMessaging_Publis
 
 		m := &messaging_pb.Message{
 			Timestamp: time.Now().UnixNano(),
-			Key:       in.Key,
-			Value:     in.Value,
-			Headers:   in.Headers,
+			Key:       in.Data.Key,
+			Value:     in.Data.Value,
+			Headers:   in.Data.Headers,
 		}
 
 		data, err := proto.Marshal(m)
@@ -84,7 +86,7 @@ func (broker *MessageBroker) Publish(stream messaging_pb.SeaweedMessaging_Publis
 			continue
 		}
 
-		logBuffer.AddToBuffer(in.Key, data)
+		logBuffer.AddToBuffer(in.Data.Key, data)
 
 	}
 }
diff --git a/weed/pb/messaging.proto b/weed/pb/messaging.proto
index 050c6fb17..10e317221 100644
--- a/weed/pb/messaging.proto
+++ b/weed/pb/messaging.proto
@@ -9,7 +9,7 @@ option java_outer_classname = "MessagingProto";
 
 service SeaweedMessaging {
 
-    rpc Subscribe (SubscribeRequest) returns (stream Message) {
+    rpc Subscribe (stream SubscriberMessage) returns (stream BrokerMessage) {
     }
 
     rpc Publish (stream PublishRequest) returns (stream PublishResponse) {
@@ -25,17 +25,25 @@ service SeaweedMessaging {
 
 //////////////////////////////////////////////////
 
-message SubscribeRequest {
-    string namespace = 1;
-    string topic = 2;
-    int32 partition = 3;
-    enum StartPosition {
-        LATEST = 0; // Start at the newest message
-        EARLIEST = 1; // Start at the oldest message
-        TIMESTAMP = 2; // Start after a specified timestamp, exclusive
+message SubscriberMessage {
+    message InitMessage {
+        string namespace = 1;
+        string topic = 2;
+        int32 partition = 3;
+        enum StartPosition {
+            LATEST = 0; // Start at the newest message
+            EARLIEST = 1; // Start at the oldest message
+            TIMESTAMP = 2; // Start after a specified timestamp, exclusive
+        }
+        StartPosition startPosition = 4; // Where to begin consuming from
+        int64 timestampNs = 5; // timestamp in nano seconds
+        string subscriber_id = 6; // uniquely identify a subscriber to track consumption
+    }
+    InitMessage init = 1;
+    message AckMessage {
+        int64 message_id = 1;
     }
-    StartPosition startPosition = 4; // Where to begin consuming from
-    int64 timestampNs = 5; // timestamp in nano seconds
+    AckMessage ack = 2;
 }
 
 message Message {
@@ -45,17 +53,38 @@ message Message {
     map<string, bytes> headers = 4; // Message headers
 }
 
+message BrokerMessage {
+    Message data = 1;
+    message RedirectMessage {
+        string new_broker = 1;
+    }
+    RedirectMessage redirect = 2;
+}
+
 message PublishRequest {
-    string namespace = 1; // only needed on the initial request
-    string topic = 2; // only needed on the initial request
-    int32 partition = 4;
-    bytes key = 5; // Message key
-    bytes value = 6; // Message payload
-    map<string, bytes> headers = 7; // Message headers
+    message InitMessage {
+        string namespace = 1; // only needed on the initial request
+        string topic = 2; // only needed on the initial request
+        int32 partition = 3;
+    }
+    InitMessage init = 1;
+    message DataMessage {
+        bytes key = 1; // Message key
+        bytes value = 2; // Message payload
+        map<string, bytes> headers = 3; // Message headers
+    }
+    DataMessage data = 2;
 }
 
 message PublishResponse {
-    int32 partition_count = 1;
+    message ConfigMessage {
+        int32 partition_count = 1;
+    }
+    ConfigMessage config = 1;
+    message RedirectMessage {
+        string new_broker = 1;
+    }
+    RedirectMessage redirect = 2;
 }
 
 message ConfigureTopicRequest {
diff --git a/weed/pb/messaging_pb/messaging.pb.go b/weed/pb/messaging_pb/messaging.pb.go
index 64fc7c20b..eb20e69d6 100644
--- a/weed/pb/messaging_pb/messaging.pb.go
+++ b/weed/pb/messaging_pb/messaging.pb.go
@@ -9,8 +9,9 @@ It is generated from these files:
 	messaging.proto
 
 It has these top-level messages:
-	SubscribeRequest
+	SubscriberMessage
 	Message
+	BrokerMessage
 	PublishRequest
 	PublishResponse
 	ConfigureTopicRequest
@@ -40,80 +41,130 @@ var _ = math.Inf
 // proto package needs to be updated.
 const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package
 
-type SubscribeRequest_StartPosition int32
+type SubscriberMessage_InitMessage_StartPosition int32
 
 const (
-	SubscribeRequest_LATEST    SubscribeRequest_StartPosition = 0
-	SubscribeRequest_EARLIEST  SubscribeRequest_StartPosition = 1
-	SubscribeRequest_TIMESTAMP SubscribeRequest_StartPosition = 2
+	SubscriberMessage_InitMessage_LATEST    SubscriberMessage_InitMessage_StartPosition = 0
+	SubscriberMessage_InitMessage_EARLIEST  SubscriberMessage_InitMessage_StartPosition = 1
+	SubscriberMessage_InitMessage_TIMESTAMP SubscriberMessage_InitMessage_StartPosition = 2
 )
 
-var SubscribeRequest_StartPosition_name = map[int32]string{
+var SubscriberMessage_InitMessage_StartPosition_name = map[int32]string{
 	0: "LATEST",
 	1: "EARLIEST",
 	2: "TIMESTAMP",
 }
-var SubscribeRequest_StartPosition_value = map[string]int32{
+var SubscriberMessage_InitMessage_StartPosition_value = map[string]int32{
 	"LATEST":    0,
 	"EARLIEST":  1,
 	"TIMESTAMP": 2,
 }
 
-func (x SubscribeRequest_StartPosition) String() string {
-	return proto.EnumName(SubscribeRequest_StartPosition_name, int32(x))
+func (x SubscriberMessage_InitMessage_StartPosition) String() string {
+	return proto.EnumName(SubscriberMessage_InitMessage_StartPosition_name, int32(x))
 }
-func (SubscribeRequest_StartPosition) EnumDescriptor() ([]byte, []int) {
-	return fileDescriptor0, []int{0, 0}
+func (SubscriberMessage_InitMessage_StartPosition) EnumDescriptor() ([]byte, []int) {
+	return fileDescriptor0, []int{0, 0, 0}
+}
+
+type SubscriberMessage struct {
+	Init *SubscriberMessage_InitMessage `protobuf:"bytes,1,opt,name=init" json:"init,omitempty"`
+	Ack  *SubscriberMessage_AckMessage  `protobuf:"bytes,2,opt,name=ack" json:"ack,omitempty"`
+}
+
+func (m *SubscriberMessage) Reset()                    { *m = SubscriberMessage{} }
+func (m *SubscriberMessage) String() string            { return proto.CompactTextString(m) }
+func (*SubscriberMessage) ProtoMessage()               {}
+func (*SubscriberMessage) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{0} }
+
+func (m *SubscriberMessage) GetInit() *SubscriberMessage_InitMessage {
+	if m != nil {
+		return m.Init
+	}
+	return nil
+}
+
+func (m *SubscriberMessage) GetAck() *SubscriberMessage_AckMessage {
+	if m != nil {
+		return m.Ack
+	}
+	return nil
 }
 
-type SubscribeRequest struct {
-	Namespace     string                         `protobuf:"bytes,1,opt,name=namespace" json:"namespace,omitempty"`
-	Topic         string                         `protobuf:"bytes,2,opt,name=topic" json:"topic,omitempty"`
-	Partition     int32                          `protobuf:"varint,3,opt,name=partition" json:"partition,omitempty"`
-	StartPosition SubscribeRequest_StartPosition `protobuf:"varint,4,opt,name=startPosition,enum=messaging_pb.SubscribeRequest_StartPosition" json:"startPosition,omitempty"`
-	TimestampNs   int64                          `protobuf:"varint,5,opt,name=timestampNs" json:"timestampNs,omitempty"`
+type SubscriberMessage_InitMessage struct {
+	Namespace     string                                      `protobuf:"bytes,1,opt,name=namespace" json:"namespace,omitempty"`
+	Topic         string                                      `protobuf:"bytes,2,opt,name=topic" json:"topic,omitempty"`
+	Partition     int32                                       `protobuf:"varint,3,opt,name=partition" json:"partition,omitempty"`
+	StartPosition SubscriberMessage_InitMessage_StartPosition `protobuf:"varint,4,opt,name=startPosition,enum=messaging_pb.SubscriberMessage_InitMessage_StartPosition" json:"startPosition,omitempty"`
+	TimestampNs   int64                                       `protobuf:"varint,5,opt,name=timestampNs" json:"timestampNs,omitempty"`
+	SubscriberId  string                                      `protobuf:"bytes,6,opt,name=subscriber_id,json=subscriberId" json:"subscriber_id,omitempty"`
 }
 
-func (m *SubscribeRequest) Reset()                    { *m = SubscribeRequest{} }
-func (m *SubscribeRequest) String() string            { return proto.CompactTextString(m) }
-func (*SubscribeRequest) ProtoMessage()               {}
-func (*SubscribeRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{0} }
+func (m *SubscriberMessage_InitMessage) Reset()         { *m = SubscriberMessage_InitMessage{} }
+func (m *SubscriberMessage_InitMessage) String() string { return proto.CompactTextString(m) }
+func (*SubscriberMessage_InitMessage) ProtoMessage()    {}
+func (*SubscriberMessage_InitMessage) Descriptor() ([]byte, []int) {
+	return fileDescriptor0, []int{0, 0}
+}
 
-func (m *SubscribeRequest) GetNamespace() string {
+func (m *SubscriberMessage_InitMessage) GetNamespace() string {
 	if m != nil {
 		return m.Namespace
 	}
 	return ""
 }
 
-func (m *SubscribeRequest) GetTopic() string {
+func (m *SubscriberMessage_InitMessage) GetTopic() string {
 	if m != nil {
 		return m.Topic
 	}
 	return ""
 }
 
-func (m *SubscribeRequest) GetPartition() int32 {
+func (m *SubscriberMessage_InitMessage) GetPartition() int32 {
 	if m != nil {
 		return m.Partition
 	}
 	return 0
 }
 
-func (m *SubscribeRequest) GetStartPosition() SubscribeRequest_StartPosition {
+func (m *SubscriberMessage_InitMessage) GetStartPosition() SubscriberMessage_InitMessage_StartPosition {
 	if m != nil {
 		return m.StartPosition
 	}
-	return SubscribeRequest_LATEST
+	return SubscriberMessage_InitMessage_LATEST
 }
 
-func (m *SubscribeRequest) GetTimestampNs() int64 {
+func (m *SubscriberMessage_InitMessage) GetTimestampNs() int64 {
 	if m != nil {
 		return m.TimestampNs
 	}
 	return 0
 }
 
+func (m *SubscriberMessage_InitMessage) GetSubscriberId() string {
+	if m != nil {
+		return m.SubscriberId
+	}
+	return ""
+}
+
+type SubscriberMessage_AckMessage struct {
+	MessageId int64 `protobuf:"varint,1,opt,name=message_id,json=messageId" json:"message_id,omitempty"`
+}
+
+func (m *SubscriberMessage_AckMessage) Reset()                    { *m = SubscriberMessage_AckMessage{} }
+func (m *SubscriberMessage_AckMessage) String() string            { return proto.CompactTextString(m) }
+func (*SubscriberMessage_AckMessage) ProtoMessage()               {}
+func (*SubscriberMessage_AckMessage) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{0, 1} }
+
+func (m *SubscriberMessage_AckMessage) GetMessageId() int64 {
+	if m != nil {
+		return m.MessageId
+	}
+	return 0
+}
+
 type Message struct {
 	Timestamp int64             `protobuf:"varint,1,opt,name=timestamp" json:"timestamp,omitempty"`
 	Key       []byte            `protobuf:"bytes,2,opt,name=key,proto3" json:"key,omitempty"`
@@ -154,56 +205,130 @@ func (m *Message) GetHeaders() map[string][]byte {
 	return nil
 }
 
+type BrokerMessage struct {
+	Data     *Message                       `protobuf:"bytes,1,opt,name=data" json:"data,omitempty"`
+	Redirect *BrokerMessage_RedirectMessage `protobuf:"bytes,2,opt,name=redirect" json:"redirect,omitempty"`
+}
+
+func (m *BrokerMessage) Reset()                    { *m = BrokerMessage{} }
+func (m *BrokerMessage) String() string            { return proto.CompactTextString(m) }
+func (*BrokerMessage) ProtoMessage()               {}
+func (*BrokerMessage) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{2} }
+
+func (m *BrokerMessage) GetData() *Message {
+	if m != nil {
+		return m.Data
+	}
+	return nil
+}
+
+func (m *BrokerMessage) GetRedirect() *BrokerMessage_RedirectMessage {
+	if m != nil {
+		return m.Redirect
+	}
+	return nil
+}
+
+type BrokerMessage_RedirectMessage struct {
+	NewBroker string `protobuf:"bytes,1,opt,name=new_broker,json=newBroker" json:"new_broker,omitempty"`
+}
+
+func (m *BrokerMessage_RedirectMessage) Reset()         { *m = BrokerMessage_RedirectMessage{} }
+func (m *BrokerMessage_RedirectMessage) String() string { return proto.CompactTextString(m) }
+func (*BrokerMessage_RedirectMessage) ProtoMessage()    {}
+func (*BrokerMessage_RedirectMessage) Descriptor() ([]byte, []int) {
+	return fileDescriptor0, []int{2, 0}
+}
+
+func (m *BrokerMessage_RedirectMessage) GetNewBroker() string {
+	if m != nil {
+		return m.NewBroker
+	}
+	return ""
+}
+
 type PublishRequest struct {
-	Namespace string            `protobuf:"bytes,1,opt,name=namespace" json:"namespace,omitempty"`
-	Topic     string            `protobuf:"bytes,2,opt,name=topic" json:"topic,omitempty"`
-	Partition int32             `protobuf:"varint,4,opt,name=partition" json:"partition,omitempty"`
-	Key       []byte            `protobuf:"bytes,5,opt,name=key,proto3" json:"key,omitempty"`
-	Value     []byte            `protobuf:"bytes,6,opt,name=value,proto3" json:"value,omitempty"`
-	Headers   map[string][]byte `protobuf:"bytes,7,rep,name=headers" json:"headers,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value,proto3"`
+	Init *PublishRequest_InitMessage `protobuf:"bytes,1,opt,name=init" json:"init,omitempty"`
+	Data *PublishRequest_DataMessage `protobuf:"bytes,2,opt,name=data" json:"data,omitempty"`
 }
 
 func (m *PublishRequest) Reset()                    { *m = PublishRequest{} }
 func (m *PublishRequest) String() string            { return proto.CompactTextString(m) }
 func (*PublishRequest) ProtoMessage()               {}
-func (*PublishRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{2} }
+func (*PublishRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{3} }
 
-func (m *PublishRequest) GetNamespace() string {
+func (m *PublishRequest) GetInit() *PublishRequest_InitMessage {
+	if m != nil {
+		return m.Init
+	}
+	return nil
+}
+
+func (m *PublishRequest) GetData() *PublishRequest_DataMessage {
+	if m != nil {
+		return m.Data
+	}
+	return nil
+}
+
+type PublishRequest_InitMessage struct {
+	Namespace string `protobuf:"bytes,1,opt,name=namespace" json:"namespace,omitempty"`
+	Topic     string `protobuf:"bytes,2,opt,name=topic" json:"topic,omitempty"`
+	Partition int32  `protobuf:"varint,3,opt,name=partition" json:"partition,omitempty"`
+}
+
+func (m *PublishRequest_InitMessage) Reset()                    { *m = PublishRequest_InitMessage{} }
+func (m *PublishRequest_InitMessage) String() string            { return proto.CompactTextString(m) }
+func (*PublishRequest_InitMessage) ProtoMessage()               {}
+func (*PublishRequest_InitMessage) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{3, 0} }
+
+func (m *PublishRequest_InitMessage) GetNamespace() string {
 	if m != nil {
 		return m.Namespace
 	}
 	return ""
 }
 
-func (m *PublishRequest) GetTopic() string {
+func (m *PublishRequest_InitMessage) GetTopic() string {
 	if m != nil {
 		return m.Topic
 	}
 	return ""
 }
 
-func (m *PublishRequest) GetPartition() int32 {
+func (m *PublishRequest_InitMessage) GetPartition() int32 {
 	if m != nil {
 		return m.Partition
 	}
 	return 0
 }
 
-func (m *PublishRequest) GetKey() []byte {
+type PublishRequest_DataMessage struct {
+	Key     []byte            `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"`
+	Value   []byte            `protobuf:"bytes,2,opt,name=value,proto3" json:"value,omitempty"`
+	Headers map[string][]byte `protobuf:"bytes,3,rep,name=headers" json:"headers,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value,proto3"`
+}
+
+func (m *PublishRequest_DataMessage) Reset()                    { *m = PublishRequest_DataMessage{} }
+func (m *PublishRequest_DataMessage) String() string            { return proto.CompactTextString(m) }
+func (*PublishRequest_DataMessage) ProtoMessage()               {}
+func (*PublishRequest_DataMessage) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{3, 1} }
+
+func (m *PublishRequest_DataMessage) GetKey() []byte {
 	if m != nil {
 		return m.Key
 	}
 	return nil
 }
 
-func (m *PublishRequest) GetValue() []byte {
+func (m *PublishRequest_DataMessage) GetValue() []byte {
 	if m != nil {
 		return m.Value
 	}
 	return nil
 }
 
-func (m *PublishRequest) GetHeaders() map[string][]byte {
+func (m *PublishRequest_DataMessage) GetHeaders() map[string][]byte {
 	if m != nil {
 		return m.Headers
 	}
@@ -211,21 +336,65 @@ func (m *PublishRequest) GetHeaders() map[string][]byte {
 }
 
 type PublishResponse struct {
-	PartitionCount int32 `protobuf:"varint,1,opt,name=partition_count,json=partitionCount" json:"partition_count,omitempty"`
+	Config   *PublishResponse_ConfigMessage   `protobuf:"bytes,1,opt,name=config" json:"config,omitempty"`
+	Redirect *PublishResponse_RedirectMessage `protobuf:"bytes,2,opt,name=redirect" json:"redirect,omitempty"`
 }
 
 func (m *PublishResponse) Reset()                    { *m = PublishResponse{} }
 func (m *PublishResponse) String() string            { return proto.CompactTextString(m) }
 func (*PublishResponse) ProtoMessage()               {}
-func (*PublishResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{3} }
+func (*PublishResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{4} }
+
+func (m *PublishResponse) GetConfig() *PublishResponse_ConfigMessage {
+	if m != nil {
+		return m.Config
+	}
+	return nil
+}
+
+func (m *PublishResponse) GetRedirect() *PublishResponse_RedirectMessage {
+	if m != nil {
+		return m.Redirect
+	}
+	return nil
+}
+
+type PublishResponse_ConfigMessage struct {
+	PartitionCount int32 `protobuf:"varint,1,opt,name=partition_count,json=partitionCount" json:"partition_count,omitempty"`
+}
+
+func (m *PublishResponse_ConfigMessage) Reset()         { *m = PublishResponse_ConfigMessage{} }
+func (m *PublishResponse_ConfigMessage) String() string { return proto.CompactTextString(m) }
+func (*PublishResponse_ConfigMessage) ProtoMessage()    {}
+func (*PublishResponse_ConfigMessage) Descriptor() ([]byte, []int) {
+	return fileDescriptor0, []int{4, 0}
+}
 
-func (m *PublishResponse) GetPartitionCount() int32 {
+func (m *PublishResponse_ConfigMessage) GetPartitionCount() int32 {
 	if m != nil {
 		return m.PartitionCount
 	}
 	return 0
 }
 
+type PublishResponse_RedirectMessage struct {
+	NewBroker string `protobuf:"bytes,1,opt,name=new_broker,json=newBroker" json:"new_broker,omitempty"`
+}
+
+func (m *PublishResponse_RedirectMessage) Reset()         { *m = PublishResponse_RedirectMessage{} }
+func (m *PublishResponse_RedirectMessage) String() string { return proto.CompactTextString(m) }
+func (*PublishResponse_RedirectMessage) ProtoMessage()    {}
+func (*PublishResponse_RedirectMessage) Descriptor() ([]byte, []int) {
+	return fileDescriptor0, []int{4, 1}
+}
+
+func (m *PublishResponse_RedirectMessage) GetNewBroker() string {
+	if m != nil {
+		return m.NewBroker
+	}
+	return ""
+}
+
 type ConfigureTopicRequest struct {
 	Namespace      string `protobuf:"bytes,1,opt,name=namespace" json:"namespace,omitempty"`
 	Topic          string `protobuf:"bytes,2,opt,name=topic" json:"topic,omitempty"`
@@ -236,7 +405,7 @@ type ConfigureTopicRequest struct {
 func (m *ConfigureTopicRequest) Reset()                    { *m = ConfigureTopicRequest{} }
 func (m *ConfigureTopicRequest) String() string            { return proto.CompactTextString(m) }
 func (*ConfigureTopicRequest) ProtoMessage()               {}
-func (*ConfigureTopicRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{4} }
+func (*ConfigureTopicRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{5} }
 
 func (m *ConfigureTopicRequest) GetNamespace() string {
 	if m != nil {
@@ -272,7 +441,7 @@ type ConfigureTopicResponse struct {
 func (m *ConfigureTopicResponse) Reset()                    { *m = ConfigureTopicResponse{} }
 func (m *ConfigureTopicResponse) String() string            { return proto.CompactTextString(m) }
 func (*ConfigureTopicResponse) ProtoMessage()               {}
-func (*ConfigureTopicResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{5} }
+func (*ConfigureTopicResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{6} }
 
 type GetTopicConfigurationRequest struct {
 	Namespace string `protobuf:"bytes,1,opt,name=namespace" json:"namespace,omitempty"`
@@ -282,7 +451,7 @@ type GetTopicConfigurationRequest struct {
 func (m *GetTopicConfigurationRequest) Reset()                    { *m = GetTopicConfigurationRequest{} }
 func (m *GetTopicConfigurationRequest) String() string            { return proto.CompactTextString(m) }
 func (*GetTopicConfigurationRequest) ProtoMessage()               {}
-func (*GetTopicConfigurationRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{6} }
+func (*GetTopicConfigurationRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{7} }
 
 func (m *GetTopicConfigurationRequest) GetNamespace() string {
 	if m != nil {
@@ -305,7 +474,7 @@ type GetTopicConfigurationResponse struct {
 func (m *GetTopicConfigurationResponse) Reset()                    { *m = GetTopicConfigurationResponse{} }
 func (m *GetTopicConfigurationResponse) String() string            { return proto.CompactTextString(m) }
 func (*GetTopicConfigurationResponse) ProtoMessage()               {}
-func (*GetTopicConfigurationResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{7} }
+func (*GetTopicConfigurationResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{8} }
 
 func (m *GetTopicConfigurationResponse) GetPartitions() int32 {
 	if m != nil {
@@ -315,15 +484,23 @@ func (m *GetTopicConfigurationResponse) GetPartitions() int32 {
 }
 
 func init() {
-	proto.RegisterType((*SubscribeRequest)(nil), "messaging_pb.SubscribeRequest")
+	proto.RegisterType((*SubscriberMessage)(nil), "messaging_pb.SubscriberMessage")
+	proto.RegisterType((*SubscriberMessage_InitMessage)(nil), "messaging_pb.SubscriberMessage.InitMessage")
+	proto.RegisterType((*SubscriberMessage_AckMessage)(nil), "messaging_pb.SubscriberMessage.AckMessage")
 	proto.RegisterType((*Message)(nil), "messaging_pb.Message")
+	proto.RegisterType((*BrokerMessage)(nil), "messaging_pb.BrokerMessage")
+	proto.RegisterType((*BrokerMessage_RedirectMessage)(nil), "messaging_pb.BrokerMessage.RedirectMessage")
 	proto.RegisterType((*PublishRequest)(nil), "messaging_pb.PublishRequest")
+	proto.RegisterType((*PublishRequest_InitMessage)(nil), "messaging_pb.PublishRequest.InitMessage")
+	proto.RegisterType((*PublishRequest_DataMessage)(nil), "messaging_pb.PublishRequest.DataMessage")
 	proto.RegisterType((*PublishResponse)(nil), "messaging_pb.PublishResponse")
+	proto.RegisterType((*PublishResponse_ConfigMessage)(nil), "messaging_pb.PublishResponse.ConfigMessage")
+	proto.RegisterType((*PublishResponse_RedirectMessage)(nil), "messaging_pb.PublishResponse.RedirectMessage")
 	proto.RegisterType((*ConfigureTopicRequest)(nil), "messaging_pb.ConfigureTopicRequest")
 	proto.RegisterType((*ConfigureTopicResponse)(nil), "messaging_pb.ConfigureTopicResponse")
 	proto.RegisterType((*GetTopicConfigurationRequest)(nil), "messaging_pb.GetTopicConfigurationRequest")
 	proto.RegisterType((*GetTopicConfigurationResponse)(nil), "messaging_pb.GetTopicConfigurationResponse")
-	proto.RegisterEnum("messaging_pb.SubscribeRequest_StartPosition", SubscribeRequest_StartPosition_name, SubscribeRequest_StartPosition_value)
+	proto.RegisterEnum("messaging_pb.SubscriberMessage_InitMessage_StartPosition", SubscriberMessage_InitMessage_StartPosition_name, SubscriberMessage_InitMessage_StartPosition_value)
 }
 
 // Reference imports to suppress errors if they are not otherwise used.
@@ -337,7 +514,7 @@ const _ = grpc.SupportPackageIsVersion4
 // Client API for SeaweedMessaging service
 
 type SeaweedMessagingClient interface {
-	Subscribe(ctx context.Context, in *SubscribeRequest, opts ...grpc.CallOption) (SeaweedMessaging_SubscribeClient, error)
+	Subscribe(ctx context.Context, opts ...grpc.CallOption) (SeaweedMessaging_SubscribeClient, error)
 	Publish(ctx context.Context, opts ...grpc.CallOption) (SeaweedMessaging_PublishClient, error)
 	ConfigureTopic(ctx context.Context, in *ConfigureTopicRequest, opts ...grpc.CallOption) (*ConfigureTopicResponse, error)
 	GetTopicConfiguration(ctx context.Context, in *GetTopicConfigurationRequest, opts ...grpc.CallOption) (*GetTopicConfigurationResponse, error)
@@ -351,23 +528,18 @@ func NewSeaweedMessagingClient(cc *grpc.ClientConn) SeaweedMessagingClient {
 	return &seaweedMessagingClient{cc}
 }
 
-func (c *seaweedMessagingClient) Subscribe(ctx context.Context, in *SubscribeRequest, opts ...grpc.CallOption) (SeaweedMessaging_SubscribeClient, error) {
+func (c *seaweedMessagingClient) Subscribe(ctx context.Context, opts ...grpc.CallOption) (SeaweedMessaging_SubscribeClient, error) {
 	stream, err := grpc.NewClientStream(ctx, &_SeaweedMessaging_serviceDesc.Streams[0], c.cc, "/messaging_pb.SeaweedMessaging/Subscribe", opts...)
 	if err != nil {
 		return nil, err
 	}
 	x := &seaweedMessagingSubscribeClient{stream}
-	if err := x.ClientStream.SendMsg(in); err != nil {
-		return nil, err
-	}
-	if err := x.ClientStream.CloseSend(); err != nil {
-		return nil, err
-	}
 	return x, nil
 }
 
 type SeaweedMessaging_SubscribeClient interface {
-	Recv() (*Message, error)
+	Send(*SubscriberMessage) error
+	Recv() (*BrokerMessage, error)
 	grpc.ClientStream
 }
 
@@ -375,8 +547,12 @@ type seaweedMessagingSubscribeClient struct {
 	grpc.ClientStream
 }
 
-func (x *seaweedMessagingSubscribeClient) Recv() (*Message, error) {
-	m := new(Message)
+func (x *seaweedMessagingSubscribeClient) Send(m *SubscriberMessage) error {
+	return x.ClientStream.SendMsg(m)
+}
+
+func (x *seaweedMessagingSubscribeClient) Recv() (*BrokerMessage, error) {
+	m := new(BrokerMessage)
 	if err := x.ClientStream.RecvMsg(m); err != nil {
 		return nil, err
 	}
@@ -435,7 +611,7 @@ func (c *seaweedMessagingClient) GetTopicConfiguration(ctx context.Context, in *
 // Server API for SeaweedMessaging service
 
 type SeaweedMessagingServer interface {
-	Subscribe(*SubscribeRequest, SeaweedMessaging_SubscribeServer) error
+	Subscribe(SeaweedMessaging_SubscribeServer) error
 	Publish(SeaweedMessaging_PublishServer) error
 	ConfigureTopic(context.Context, *ConfigureTopicRequest) (*ConfigureTopicResponse, error)
 	GetTopicConfiguration(context.Context, *GetTopicConfigurationRequest) (*GetTopicConfigurationResponse, error)
@@ -446,15 +622,12 @@ func RegisterSeaweedMessagingServer(s *grpc.Server, srv SeaweedMessagingServer)
 }
 
 func _SeaweedMessaging_Subscribe_Handler(srv interface{}, stream grpc.ServerStream) error {
-	m := new(SubscribeRequest)
-	if err := stream.RecvMsg(m); err != nil {
-		return err
-	}
-	return srv.(SeaweedMessagingServer).Subscribe(m, &seaweedMessagingSubscribeServer{stream})
+	return srv.(SeaweedMessagingServer).Subscribe(&seaweedMessagingSubscribeServer{stream})
 }
 
 type SeaweedMessaging_SubscribeServer interface {
-	Send(*Message) error
+	Send(*BrokerMessage) error
+	Recv() (*SubscriberMessage, error)
 	grpc.ServerStream
 }
 
@@ -462,10 +635,18 @@ type seaweedMessagingSubscribeServer struct {
 	grpc.ServerStream
 }
 
-func (x *seaweedMessagingSubscribeServer) Send(m *Message) error {
+func (x *seaweedMessagingSubscribeServer) Send(m *BrokerMessage) error {
 	return x.ServerStream.SendMsg(m)
 }
 
+func (x *seaweedMessagingSubscribeServer) Recv() (*SubscriberMessage, error) {
+	m := new(SubscriberMessage)
+	if err := x.ServerStream.RecvMsg(m); err != nil {
+		return nil, err
+	}
+	return m, nil
+}
+
 func _SeaweedMessaging_Publish_Handler(srv interface{}, stream grpc.ServerStream) error {
 	return srv.(SeaweedMessagingServer).Publish(&seaweedMessagingPublishServer{stream})
 }
@@ -546,6 +727,7 @@ var _SeaweedMessaging_serviceDesc = grpc.ServiceDesc{
 			StreamName:    "Subscribe",
 			Handler:       _SeaweedMessaging_Subscribe_Handler,
 			ServerStreams: true,
+			ClientStreams: true,
 		},
 		{
 			StreamName:    "Publish",
@@ -560,42 +742,56 @@ var _SeaweedMessaging_serviceDesc = grpc.ServiceDesc{
 func init() { proto.RegisterFile("messaging.proto", fileDescriptor0) }
 
 var fileDescriptor0 = []byte{
-	// 586 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xb4, 0x95, 0xdb, 0x8b, 0xda, 0x4e,
-	0x14, 0xc7, 0x9d, 0xc4, 0xcb, 0x2f, 0x67, 0xbd, 0x31, 0xfc, 0x2c, 0x41, 0x5c, 0x09, 0x69, 0xa1,
-	0xe9, 0x85, 0x20, 0xf6, 0x65, 0x91, 0x42, 0x51, 0xb1, 0xed, 0x82, 0x16, 0x19, 0x7d, 0x2d, 0x4b,
-	0xcc, 0xce, 0xba, 0xa1, 0x9a, 0xa4, 0x99, 0x49, 0xcb, 0xfe, 0x0d, 0x7d, 0xdd, 0x7f, 0xab, 0x7f,
-	0x51, 0x5f, 0x4a, 0xae, 0x26, 0x92, 0x95, 0xd2, 0xcb, 0x5b, 0xe6, 0x3b, 0x67, 0xce, 0xf7, 0x9c,
-	0xcf, 0x49, 0x26, 0xd0, 0xda, 0x53, 0xc6, 0x8c, 0xad, 0x65, 0x6f, 0x75, 0xd7, 0x73, 0xb8, 0x83,
-	0xeb, 0xa9, 0x70, 0xe5, 0x6e, 0xd4, 0x7b, 0x01, 0xda, 0x2b, 0x7f, 0xc3, 0x4c, 0xcf, 0xda, 0x50,
-	0x42, 0x3f, 0xfb, 0x94, 0x71, 0xdc, 0x03, 0xc9, 0x36, 0xf6, 0x94, 0xb9, 0x86, 0x49, 0x65, 0xa4,
-	0x20, 0x4d, 0x22, 0x07, 0x01, 0xff, 0x0f, 0x15, 0xee, 0xb8, 0x96, 0x29, 0x0b, 0xe1, 0x4e, 0xb4,
-	0x08, 0xce, 0xb8, 0x86, 0xc7, 0x2d, 0x6e, 0x39, 0xb6, 0x2c, 0x2a, 0x48, 0xab, 0x90, 0x83, 0x80,
-	0x09, 0x34, 0x18, 0x37, 0x3c, 0xbe, 0x74, 0x58, 0x14, 0x51, 0x56, 0x90, 0xd6, 0x1c, 0xbe, 0xd4,
-	0xb3, 0xc5, 0xe8, 0xc7, 0x85, 0xe8, 0xab, 0xec, 0x19, 0x92, 0x4f, 0x81, 0x15, 0x38, 0xe3, 0xd6,
-	0x9e, 0x32, 0x6e, 0xec, 0xdd, 0x0f, 0x4c, 0xae, 0x28, 0x48, 0x13, 0x49, 0x56, 0x52, 0x2f, 0xa0,
-	0x91, 0xcb, 0x80, 0x01, 0xaa, 0xf3, 0xf1, 0x7a, 0xb6, 0x5a, 0xb7, 0x4b, 0xb8, 0x0e, 0xff, 0xcd,
-	0xc6, 0x64, 0x7e, 0x19, 0xac, 0x10, 0x6e, 0x80, 0xb4, 0xbe, 0x5c, 0xcc, 0x56, 0xeb, 0xf1, 0x62,
-	0xd9, 0x16, 0xd4, 0xef, 0x08, 0x6a, 0x8b, 0xb0, 0x34, 0x8a, 0x15, 0x90, 0xd2, 0xa4, 0x21, 0x0d,
-	0x71, 0x22, 0x0c, 0x10, 0x39, 0x88, 0xb8, 0x0d, 0xe2, 0x27, 0x7a, 0x17, 0xf2, 0xa8, 0x93, 0xe0,
-	0x31, 0x60, 0xf4, 0xc5, 0xd8, 0xf9, 0x34, 0x24, 0x51, 0x27, 0xd1, 0x02, 0xbf, 0x86, 0xda, 0x2d,
-	0x35, 0xae, 0xa9, 0xc7, 0xe4, 0xb2, 0x22, 0x6a, 0x67, 0x43, 0x35, 0xdf, 0x7f, 0xec, 0xa8, 0xbf,
-	0x8f, 0x82, 0x66, 0x36, 0xf7, 0xee, 0x48, 0x72, 0xa4, 0x3b, 0x82, 0x7a, 0x76, 0x23, 0x71, 0x8d,
-	0xe6, 0x93, 0x77, 0x15, 0x32, 0xae, 0x23, 0xe1, 0x02, 0xa9, 0xdf, 0x04, 0x68, 0x2e, 0xfd, 0xcd,
-	0xce, 0x62, 0xb7, 0x7f, 0x6d, 0xc8, 0xe5, 0xe3, 0x21, 0xc7, 0x05, 0x55, 0x0a, 0x30, 0x54, 0xb3,
-	0x18, 0xa6, 0x07, 0x0c, 0xb5, 0x10, 0xc3, 0xb3, 0x3c, 0x86, 0x7c, 0xa1, 0xff, 0x80, 0xc6, 0x08,
-	0x5a, 0xa9, 0x07, 0x73, 0x1d, 0x9b, 0x51, 0xfc, 0x14, 0x5a, 0x69, 0x23, 0x57, 0xa6, 0xe3, 0xdb,
-	0x3c, 0x4c, 0x55, 0x21, 0xcd, 0x54, 0x9e, 0x06, 0xaa, 0x7a, 0x8f, 0xa0, 0x33, 0x75, 0xec, 0x1b,
-	0x6b, 0xeb, 0x7b, 0x74, 0x1d, 0x50, 0xf9, 0x13, 0xa0, 0x05, 0xb6, 0x62, 0x91, 0x2d, 0xee, 0x03,
-	0x98, 0xce, 0x6e, 0x47, 0xcd, 0x14, 0xbd, 0x44, 0x32, 0x8a, 0x2a, 0xc3, 0xa3, 0xe3, 0xaa, 0xa2,
-	0xce, 0x54, 0x02, 0xbd, 0x77, 0x94, 0x87, 0x5a, 0x12, 0x61, 0x84, 0x9f, 0xd3, 0xef, 0x97, 0xad,
-	0xbe, 0x81, 0xf3, 0x07, 0x72, 0xc6, 0x38, 0xfb, 0x00, 0x69, 0x03, 0x2c, 0x6e, 0x29, 0xa3, 0x0c,
-	0x7f, 0x04, 0xd7, 0x0e, 0x35, 0xbe, 0x52, 0x7a, 0xbd, 0x48, 0x46, 0x8f, 0xdf, 0x82, 0x94, 0xde,
-	0x00, 0xb8, 0x7f, 0xfa, 0x6a, 0xe8, 0x76, 0x0a, 0x3f, 0x1d, 0xb5, 0x34, 0x40, 0x78, 0x0e, 0xb5,
-	0x78, 0xbc, 0xb8, 0x77, 0xea, 0xcd, 0xea, 0x9e, 0x3f, 0xb0, 0x1b, 0x93, 0x2b, 0x69, 0x68, 0x80,
-	0xf0, 0x47, 0x68, 0xe6, 0xc9, 0xe2, 0xc7, 0xf9, 0x63, 0x85, 0x6f, 0x43, 0xf7, 0xc9, 0xe9, 0xa0,
-	0xc4, 0x02, 0x7b, 0xd0, 0x29, 0x44, 0x89, 0x9f, 0xe7, 0x13, 0x9c, 0x9a, 0x61, 0xf7, 0xc5, 0x2f,
-	0xc5, 0x26, 0x9e, 0x13, 0x15, 0xda, 0x2c, 0x82, 0x7f, 0xc3, 0x74, 0x73, 0x67, 0x51, 0x9b, 0x4f,
-	0x9a, 0xe9, 0x1c, 0x96, 0xc1, 0x6f, 0x62, 0x53, 0x0d, 0xff, 0x16, 0xaf, 0x7e, 0x06, 0x00, 0x00,
-	0xff, 0xff, 0x83, 0x4f, 0x5b, 0x91, 0x40, 0x06, 0x00, 0x00,
+	// 802 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xb4, 0x56, 0x5d, 0x4f, 0xdb, 0x48,
+	0x14, 0xc5, 0x76, 0x12, 0xf0, 0xcd, 0xe7, 0x8e, 0x96, 0x55, 0xe4, 0x05, 0x36, 0x32, 0x2b, 0x6d,
+	0xb6, 0xa8, 0x16, 0x4a, 0x55, 0x89, 0x22, 0x24, 0x94, 0xd0, 0x88, 0x46, 0x22, 0x25, 0x9a, 0xe4,
+	0xb5, 0x8a, 0x26, 0xce, 0x00, 0x56, 0x12, 0x3b, 0xf5, 0x4c, 0x8a, 0x78, 0x6e, 0x5f, 0xfb, 0xd6,
+	0x5f, 0x52, 0xa9, 0x3f, 0xa0, 0x7d, 0xef, 0x7f, 0xaa, 0x3c, 0xfe, 0x88, 0x1d, 0x42, 0xa0, 0x48,
+	0xbc, 0xd9, 0xd7, 0xe7, 0xdc, 0x73, 0xe7, 0xde, 0x33, 0xe3, 0x81, 0xe2, 0x84, 0x32, 0x46, 0x2e,
+	0x2d, 0xfb, 0xd2, 0x98, 0xba, 0x0e, 0x77, 0x50, 0x2e, 0x0a, 0xf4, 0xa7, 0x03, 0xfd, 0x63, 0x0a,
+	0xfe, 0xe8, 0xce, 0x06, 0xcc, 0x74, 0xad, 0x01, 0x75, 0xdb, 0xe2, 0x13, 0x45, 0xc7, 0x90, 0xb2,
+	0x6c, 0x8b, 0x97, 0xa5, 0x8a, 0x54, 0xcd, 0xd6, 0xf6, 0x8c, 0x38, 0xc5, 0xb8, 0x05, 0x37, 0x5a,
+	0xb6, 0xc5, 0x83, 0x67, 0x2c, 0x88, 0xe8, 0x08, 0x14, 0x62, 0x8e, 0xca, 0xb2, 0xe0, 0x3f, 0xbb,
+	0x8f, 0x5f, 0x37, 0x47, 0x21, 0xdd, 0xa3, 0x69, 0xdf, 0x65, 0xc8, 0xc6, 0x72, 0xa2, 0x2d, 0x50,
+	0x6d, 0x32, 0xa1, 0x6c, 0x4a, 0x4c, 0x2a, 0x6a, 0x52, 0xf1, 0x3c, 0x80, 0xfe, 0x84, 0x34, 0x77,
+	0xa6, 0x96, 0x29, 0xd4, 0x54, 0xec, 0xbf, 0x78, 0x9c, 0x29, 0x71, 0xb9, 0xc5, 0x2d, 0xc7, 0x2e,
+	0x2b, 0x15, 0xa9, 0x9a, 0xc6, 0xf3, 0x00, 0xea, 0x43, 0x9e, 0x71, 0xe2, 0xf2, 0x8e, 0xc3, 0x7c,
+	0x44, 0xaa, 0x22, 0x55, 0x0b, 0xb5, 0x57, 0xbf, 0xb1, 0x52, 0xa3, 0x1b, 0x4f, 0x80, 0x93, 0xf9,
+	0x50, 0x05, 0xb2, 0xdc, 0x9a, 0x50, 0xc6, 0xc9, 0x64, 0xfa, 0x96, 0x95, 0xd3, 0x15, 0xa9, 0xaa,
+	0xe0, 0x78, 0x08, 0xed, 0x42, 0x9e, 0x45, 0xf9, 0xfb, 0xd6, 0xb0, 0x9c, 0x11, 0xe5, 0xe7, 0xe6,
+	0xc1, 0xd6, 0x50, 0x3f, 0x80, 0x7c, 0x42, 0x06, 0x01, 0x64, 0xce, 0xea, 0xbd, 0x66, 0xb7, 0x57,
+	0x5a, 0x43, 0x39, 0xd8, 0x68, 0xd6, 0xf1, 0x59, 0xcb, 0x7b, 0x93, 0x50, 0x1e, 0xd4, 0x5e, 0xab,
+	0xdd, 0xec, 0xf6, 0xea, 0xed, 0x4e, 0x49, 0xd6, 0xf6, 0x00, 0xe6, 0x6d, 0x45, 0xdb, 0x00, 0xfe,
+	0xca, 0xa8, 0xa7, 0x24, 0x89, 0x6a, 0xd4, 0x20, 0xd2, 0x1a, 0xea, 0x3f, 0x25, 0x58, 0x0f, 0xa1,
+	0x15, 0x50, 0xa3, 0x32, 0x7d, 0x64, 0x43, 0xde, 0x97, 0xf0, 0x3c, 0x88, 0x4a, 0xa0, 0x8c, 0xe8,
+	0x8d, 0x68, 0x77, 0x0e, 0x7b, 0x8f, 0xde, 0x08, 0x3e, 0x90, 0xf1, 0x8c, 0x8a, 0x46, 0xe7, 0xb0,
+	0xff, 0x82, 0x8e, 0x60, 0xfd, 0x8a, 0x92, 0x21, 0x75, 0x59, 0x39, 0x55, 0x51, 0xaa, 0xd9, 0x9a,
+	0x9e, 0x6c, 0x6f, 0xd8, 0xc8, 0x37, 0x3e, 0xa8, 0x69, 0x73, 0xf7, 0x06, 0x87, 0x14, 0xed, 0x10,
+	0x72, 0xf1, 0x0f, 0xa1, 0xaa, 0x3f, 0xfe, 0xa4, 0xaa, 0x1c, 0x53, 0x3d, 0x94, 0x0f, 0x24, 0xfd,
+	0x9b, 0x04, 0xf9, 0x86, 0xeb, 0x8c, 0xe6, 0x8e, 0xfe, 0x1f, 0x52, 0x43, 0xc2, 0x49, 0xe0, 0xe8,
+	0xcd, 0xa5, 0x85, 0x60, 0x01, 0x41, 0xa7, 0xb0, 0xe1, 0xd2, 0xa1, 0xe5, 0x52, 0x93, 0x07, 0x06,
+	0x5e, 0xd8, 0x00, 0x89, 0xcc, 0x06, 0x0e, 0xb0, 0x61, 0x92, 0x88, 0xac, 0xed, 0x43, 0x71, 0xe1,
+	0xa3, 0x37, 0x07, 0x9b, 0x5e, 0xf7, 0x07, 0x22, 0x43, 0x64, 0x65, 0x7a, 0xed, 0xa7, 0xd4, 0xbf,
+	0x2a, 0x50, 0xe8, 0xcc, 0x06, 0x63, 0x8b, 0x5d, 0x61, 0xfa, 0x7e, 0x46, 0x99, 0xb7, 0x93, 0xe2,
+	0x5b, 0xb1, 0x9a, 0xac, 0x24, 0x89, 0x5d, 0xba, 0x0f, 0xfd, 0x65, 0xcb, 0x0f, 0x60, 0xbf, 0x26,
+	0x9c, 0x24, 0x3a, 0xa1, 0xf5, 0x9f, 0x78, 0x1b, 0x6a, 0x3f, 0x24, 0xc8, 0xc6, 0x64, 0xe3, 0x33,
+	0xce, 0xad, 0x98, 0x31, 0x3a, 0x9f, 0x3b, 0x4b, 0x11, 0xce, 0x7a, 0xf9, 0xd0, 0x95, 0x3d, 0x81,
+	0xd9, 0x3e, 0xcb, 0x50, 0x8c, 0x04, 0xd9, 0xd4, 0xb1, 0x19, 0x45, 0x27, 0x90, 0x31, 0x1d, 0xfb,
+	0xc2, 0xba, 0x5c, 0x7e, 0x84, 0x2e, 0xc0, 0x8d, 0x13, 0x81, 0x0d, 0x9b, 0x1f, 0x50, 0x51, 0xeb,
+	0x96, 0x11, 0x9f, 0xaf, 0x4e, 0x73, 0xb7, 0x15, 0x0f, 0x20, 0x9f, 0xd0, 0x40, 0xff, 0x41, 0x31,
+	0x1a, 0x43, 0xdf, 0x74, 0x66, 0xb6, 0xef, 0xb0, 0x34, 0x2e, 0x44, 0xe1, 0x13, 0x2f, 0xfa, 0x08,
+	0x13, 0x7f, 0x91, 0x60, 0xd3, 0x17, 0x9b, 0xb9, 0xb4, 0xe7, 0xb9, 0x20, 0xf4, 0xf2, 0x63, 0x0c,
+	0xb4, 0xa4, 0x50, 0x65, 0x59, 0xa1, 0x68, 0x07, 0xc0, 0x74, 0xc6, 0x63, 0x6a, 0x46, 0xe7, 0xb9,
+	0x8a, 0x63, 0x11, 0xbd, 0x0c, 0x7f, 0x2d, 0x56, 0xe5, 0xb7, 0x4d, 0xc7, 0xb0, 0x75, 0x4a, 0xb9,
+	0x88, 0x85, 0x08, 0x22, 0xce, 0xf4, 0xc7, 0x97, 0xad, 0x1f, 0xc3, 0xf6, 0x1d, 0x39, 0x03, 0x87,
+	0xec, 0x00, 0x44, 0x0b, 0x60, 0xc1, 0x92, 0x62, 0x91, 0xda, 0x27, 0x05, 0x4a, 0x5d, 0x4a, 0xae,
+	0x29, 0x1d, 0xb6, 0xc3, 0x99, 0xa3, 0x73, 0x50, 0xa3, 0x7f, 0x12, 0xfa, 0xe7, 0x9e, 0x9f, 0x95,
+	0xf6, 0xf7, 0x8a, 0x63, 0x4b, 0x5f, 0xab, 0x4a, 0xfb, 0x12, 0x3a, 0x83, 0xf5, 0xc0, 0x44, 0x68,
+	0x6b, 0xd5, 0x16, 0xd2, 0xb6, 0x57, 0x3a, 0x2f, 0xc8, 0xf6, 0x0e, 0x0a, 0xc9, 0x16, 0xa3, 0xdd,
+	0x24, 0x6d, 0xa9, 0x2d, 0xb4, 0x7f, 0x57, 0x83, 0x42, 0x09, 0xe4, 0xc2, 0xe6, 0xd2, 0x9e, 0xa2,
+	0x85, 0x0b, 0xc6, 0xaa, 0x61, 0x6a, 0x7b, 0x0f, 0xc2, 0x86, 0x9a, 0x0d, 0x1d, 0x4a, 0xcc, 0x9f,
+	0xc2, 0x05, 0x33, 0xcc, 0xb1, 0x45, 0x6d, 0xde, 0x28, 0x44, 0x03, 0xe9, 0x78, 0x37, 0xaa, 0x41,
+	0x46, 0x5c, 0xac, 0x5e, 0xfc, 0x0a, 0x00, 0x00, 0xff, 0xff, 0x5f, 0x31, 0x28, 0xa2, 0x6b, 0x09,
+	0x00, 0x00,
 }

From 7d3672c60a8ee9fc7f540d36f410c81298f8439d Mon Sep 17 00:00:00 2001
From: Chris Lu <chris.lu@gmail.com>
Date: Fri, 17 Apr 2020 00:00:48 -0700
Subject: [PATCH 71/81] simplify

---
 weed/filer2/filer_notify_append.go | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/weed/filer2/filer_notify_append.go b/weed/filer2/filer_notify_append.go
index 0e6e8d50f..62cfd2cd1 100644
--- a/weed/filer2/filer_notify_append.go
+++ b/weed/filer2/filer_notify_append.go
@@ -13,7 +13,7 @@ import (
 
 func (f *Filer) appendToFile(targetFile string, data []byte) error {
 
-	assignResult, err, uploadResult, err2 := f.assignAndUpload(data)
+	assignResult, uploadResult, err2 := f.assignAndUpload(data)
 	if err2 != nil {
 		return err2
 	}
@@ -54,7 +54,7 @@ func (f *Filer) appendToFile(targetFile string, data []byte) error {
 	return err
 }
 
-func (f *Filer) assignAndUpload(data []byte) (*operation.AssignResult, error, *operation.UploadResult, error) {
+func (f *Filer) assignAndUpload(data []byte) (*operation.AssignResult, *operation.UploadResult, error) {
 	// assign a volume location
 	assignRequest := &operation.VolumeAssignRequest{
 		Count:               1,
@@ -64,18 +64,18 @@ func (f *Filer) assignAndUpload(data []byte) (*operation.AssignResult, error, *o
 	}
 	assignResult, err := operation.Assign(f.GetMaster(), f.GrpcDialOption, assignRequest)
 	if err != nil {
-		return nil, nil, nil, fmt.Errorf("AssignVolume: %v", err)
+		return nil, nil, fmt.Errorf("AssignVolume: %v", err)
 	}
 	if assignResult.Error != "" {
-		return nil, nil, nil, fmt.Errorf("AssignVolume error: %v", assignResult.Error)
+		return nil, nil, fmt.Errorf("AssignVolume error: %v", assignResult.Error)
 	}
 
 	// upload data
 	targetUrl := "http://" + assignResult.Url + "/" + assignResult.Fid
 	uploadResult, err := operation.UploadData(targetUrl, "", false, data, false, "", nil, assignResult.Auth)
 	if err != nil {
-		return nil, nil, nil, fmt.Errorf("upload data %s: %v", targetUrl, err)
+		return nil, nil, fmt.Errorf("upload data %s: %v", targetUrl, err)
 	}
 	// println("uploaded to", targetUrl)
-	return assignResult, err, uploadResult, nil
+	return assignResult, uploadResult, nil
 }

From 722b14337eefc69f955a2189c5fc7dff4525eac0 Mon Sep 17 00:00:00 2001
From: Chris Lu <chris.lu@gmail.com>
Date: Fri, 17 Apr 2020 01:26:27 -0700
Subject: [PATCH 72/81] add cipher option to meta data updates

---
 weed/filer2/filer_notify_append.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/weed/filer2/filer_notify_append.go b/weed/filer2/filer_notify_append.go
index 62cfd2cd1..6671cd909 100644
--- a/weed/filer2/filer_notify_append.go
+++ b/weed/filer2/filer_notify_append.go
@@ -72,7 +72,7 @@ func (f *Filer) assignAndUpload(data []byte) (*operation.AssignResult, *operatio
 
 	// upload data
 	targetUrl := "http://" + assignResult.Url + "/" + assignResult.Fid
-	uploadResult, err := operation.UploadData(targetUrl, "", false, data, false, "", nil, assignResult.Auth)
+	uploadResult, err := operation.UploadData(targetUrl, "", f.Cipher, data, false, "", nil, assignResult.Auth)
 	if err != nil {
 		return nil, nil, fmt.Errorf("upload data %s: %v", targetUrl, err)
 	}

From bda82f61bc847d6d02ebd9b242c07e01588b4e30 Mon Sep 17 00:00:00 2001
From: Chris Lu <chris.lu@gmail.com>
Date: Fri, 17 Apr 2020 02:28:09 -0700
Subject: [PATCH 73/81] filer: able to append to a file

---
 other/java/client/src/main/proto/filer.proto |  11 +
 weed/pb/filer.proto                          |  11 +
 weed/pb/filer_pb/filer.pb.go                 | 359 ++++++++++++-------
 weed/server/filer_grpc_server.go             |  32 ++
 4 files changed, 274 insertions(+), 139 deletions(-)

diff --git a/other/java/client/src/main/proto/filer.proto b/other/java/client/src/main/proto/filer.proto
index fd2b8ebe3..bc159fd14 100644
--- a/other/java/client/src/main/proto/filer.proto
+++ b/other/java/client/src/main/proto/filer.proto
@@ -21,6 +21,9 @@ service SeaweedFiler {
     rpc UpdateEntry (UpdateEntryRequest) returns (UpdateEntryResponse) {
     }
 
+    rpc AppendToEntry (AppendToEntryRequest) returns (AppendToEntryResponse) {
+    }
+
     rpc DeleteEntry (DeleteEntryRequest) returns (DeleteEntryResponse) {
     }
 
@@ -143,6 +146,14 @@ message UpdateEntryRequest {
 message UpdateEntryResponse {
 }
 
+message AppendToEntryRequest {
+    string directory = 1;
+    string entry_name = 2;
+    repeated FileChunk chunks = 3;
+}
+message AppendToEntryResponse {
+}
+
 message DeleteEntryRequest {
     string directory = 1;
     string name = 2;
diff --git a/weed/pb/filer.proto b/weed/pb/filer.proto
index fd2b8ebe3..bc159fd14 100644
--- a/weed/pb/filer.proto
+++ b/weed/pb/filer.proto
@@ -21,6 +21,9 @@ service SeaweedFiler {
     rpc UpdateEntry (UpdateEntryRequest) returns (UpdateEntryResponse) {
     }
 
+    rpc AppendToEntry (AppendToEntryRequest) returns (AppendToEntryResponse) {
+    }
+
     rpc DeleteEntry (DeleteEntryRequest) returns (DeleteEntryResponse) {
     }
 
@@ -143,6 +146,14 @@ message UpdateEntryRequest {
 message UpdateEntryResponse {
 }
 
+message AppendToEntryRequest {
+    string directory = 1;
+    string entry_name = 2;
+    repeated FileChunk chunks = 3;
+}
+message AppendToEntryResponse {
+}
+
 message DeleteEntryRequest {
     string directory = 1;
     string name = 2;
diff --git a/weed/pb/filer_pb/filer.pb.go b/weed/pb/filer_pb/filer.pb.go
index 2454395b4..2f254d22f 100644
--- a/weed/pb/filer_pb/filer.pb.go
+++ b/weed/pb/filer_pb/filer.pb.go
@@ -23,6 +23,8 @@ It has these top-level messages:
 	CreateEntryResponse
 	UpdateEntryRequest
 	UpdateEntryResponse
+	AppendToEntryRequest
+	AppendToEntryResponse
 	DeleteEntryRequest
 	DeleteEntryResponse
 	AtomicRenameEntryRequest
@@ -601,6 +603,46 @@ func (m *UpdateEntryResponse) String() string            { return proto.CompactT
 func (*UpdateEntryResponse) ProtoMessage()               {}
 func (*UpdateEntryResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{13} }
 
+type AppendToEntryRequest struct {
+	Directory string       `protobuf:"bytes,1,opt,name=directory" json:"directory,omitempty"`
+	EntryName string       `protobuf:"bytes,2,opt,name=entry_name,json=entryName" json:"entry_name,omitempty"`
+	Chunks    []*FileChunk `protobuf:"bytes,3,rep,name=chunks" json:"chunks,omitempty"`
+}
+
+func (m *AppendToEntryRequest) Reset()                    { *m = AppendToEntryRequest{} }
+func (m *AppendToEntryRequest) String() string            { return proto.CompactTextString(m) }
+func (*AppendToEntryRequest) ProtoMessage()               {}
+func (*AppendToEntryRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{14} }
+
+func (m *AppendToEntryRequest) GetDirectory() string {
+	if m != nil {
+		return m.Directory
+	}
+	return ""
+}
+
+func (m *AppendToEntryRequest) GetEntryName() string {
+	if m != nil {
+		return m.EntryName
+	}
+	return ""
+}
+
+func (m *AppendToEntryRequest) GetChunks() []*FileChunk {
+	if m != nil {
+		return m.Chunks
+	}
+	return nil
+}
+
+type AppendToEntryResponse struct {
+}
+
+func (m *AppendToEntryResponse) Reset()                    { *m = AppendToEntryResponse{} }
+func (m *AppendToEntryResponse) String() string            { return proto.CompactTextString(m) }
+func (*AppendToEntryResponse) ProtoMessage()               {}
+func (*AppendToEntryResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{15} }
+
 type DeleteEntryRequest struct {
 	Directory string `protobuf:"bytes,1,opt,name=directory" json:"directory,omitempty"`
 	Name      string `protobuf:"bytes,2,opt,name=name" json:"name,omitempty"`
@@ -613,7 +655,7 @@ type DeleteEntryRequest struct {
 func (m *DeleteEntryRequest) Reset()                    { *m = DeleteEntryRequest{} }
 func (m *DeleteEntryRequest) String() string            { return proto.CompactTextString(m) }
 func (*DeleteEntryRequest) ProtoMessage()               {}
-func (*DeleteEntryRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{14} }
+func (*DeleteEntryRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{16} }
 
 func (m *DeleteEntryRequest) GetDirectory() string {
 	if m != nil {
@@ -657,7 +699,7 @@ type DeleteEntryResponse struct {
 func (m *DeleteEntryResponse) Reset()                    { *m = DeleteEntryResponse{} }
 func (m *DeleteEntryResponse) String() string            { return proto.CompactTextString(m) }
 func (*DeleteEntryResponse) ProtoMessage()               {}
-func (*DeleteEntryResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{15} }
+func (*DeleteEntryResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{17} }
 
 func (m *DeleteEntryResponse) GetError() string {
 	if m != nil {
@@ -676,7 +718,7 @@ type AtomicRenameEntryRequest struct {
 func (m *AtomicRenameEntryRequest) Reset()                    { *m = AtomicRenameEntryRequest{} }
 func (m *AtomicRenameEntryRequest) String() string            { return proto.CompactTextString(m) }
 func (*AtomicRenameEntryRequest) ProtoMessage()               {}
-func (*AtomicRenameEntryRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{16} }
+func (*AtomicRenameEntryRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{18} }
 
 func (m *AtomicRenameEntryRequest) GetOldDirectory() string {
 	if m != nil {
@@ -712,7 +754,7 @@ type AtomicRenameEntryResponse struct {
 func (m *AtomicRenameEntryResponse) Reset()                    { *m = AtomicRenameEntryResponse{} }
 func (m *AtomicRenameEntryResponse) String() string            { return proto.CompactTextString(m) }
 func (*AtomicRenameEntryResponse) ProtoMessage()               {}
-func (*AtomicRenameEntryResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{17} }
+func (*AtomicRenameEntryResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{19} }
 
 type AssignVolumeRequest struct {
 	Count       int32  `protobuf:"varint,1,opt,name=count" json:"count,omitempty"`
@@ -726,7 +768,7 @@ type AssignVolumeRequest struct {
 func (m *AssignVolumeRequest) Reset()                    { *m = AssignVolumeRequest{} }
 func (m *AssignVolumeRequest) String() string            { return proto.CompactTextString(m) }
 func (*AssignVolumeRequest) ProtoMessage()               {}
-func (*AssignVolumeRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{18} }
+func (*AssignVolumeRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{20} }
 
 func (m *AssignVolumeRequest) GetCount() int32 {
 	if m != nil {
@@ -784,7 +826,7 @@ type AssignVolumeResponse struct {
 func (m *AssignVolumeResponse) Reset()                    { *m = AssignVolumeResponse{} }
 func (m *AssignVolumeResponse) String() string            { return proto.CompactTextString(m) }
 func (*AssignVolumeResponse) ProtoMessage()               {}
-func (*AssignVolumeResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{19} }
+func (*AssignVolumeResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{21} }
 
 func (m *AssignVolumeResponse) GetFileId() string {
 	if m != nil {
@@ -849,7 +891,7 @@ type LookupVolumeRequest struct {
 func (m *LookupVolumeRequest) Reset()                    { *m = LookupVolumeRequest{} }
 func (m *LookupVolumeRequest) String() string            { return proto.CompactTextString(m) }
 func (*LookupVolumeRequest) ProtoMessage()               {}
-func (*LookupVolumeRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{20} }
+func (*LookupVolumeRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{22} }
 
 func (m *LookupVolumeRequest) GetVolumeIds() []string {
 	if m != nil {
@@ -865,7 +907,7 @@ type Locations struct {
 func (m *Locations) Reset()                    { *m = Locations{} }
 func (m *Locations) String() string            { return proto.CompactTextString(m) }
 func (*Locations) ProtoMessage()               {}
-func (*Locations) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{21} }
+func (*Locations) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{23} }
 
 func (m *Locations) GetLocations() []*Location {
 	if m != nil {
@@ -882,7 +924,7 @@ type Location struct {
 func (m *Location) Reset()                    { *m = Location{} }
 func (m *Location) String() string            { return proto.CompactTextString(m) }
 func (*Location) ProtoMessage()               {}
-func (*Location) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{22} }
+func (*Location) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{24} }
 
 func (m *Location) GetUrl() string {
 	if m != nil {
@@ -905,7 +947,7 @@ type LookupVolumeResponse struct {
 func (m *LookupVolumeResponse) Reset()                    { *m = LookupVolumeResponse{} }
 func (m *LookupVolumeResponse) String() string            { return proto.CompactTextString(m) }
 func (*LookupVolumeResponse) ProtoMessage()               {}
-func (*LookupVolumeResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{23} }
+func (*LookupVolumeResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{25} }
 
 func (m *LookupVolumeResponse) GetLocationsMap() map[string]*Locations {
 	if m != nil {
@@ -921,7 +963,7 @@ type DeleteCollectionRequest struct {
 func (m *DeleteCollectionRequest) Reset()                    { *m = DeleteCollectionRequest{} }
 func (m *DeleteCollectionRequest) String() string            { return proto.CompactTextString(m) }
 func (*DeleteCollectionRequest) ProtoMessage()               {}
-func (*DeleteCollectionRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{24} }
+func (*DeleteCollectionRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{26} }
 
 func (m *DeleteCollectionRequest) GetCollection() string {
 	if m != nil {
@@ -936,7 +978,7 @@ type DeleteCollectionResponse struct {
 func (m *DeleteCollectionResponse) Reset()                    { *m = DeleteCollectionResponse{} }
 func (m *DeleteCollectionResponse) String() string            { return proto.CompactTextString(m) }
 func (*DeleteCollectionResponse) ProtoMessage()               {}
-func (*DeleteCollectionResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{25} }
+func (*DeleteCollectionResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{27} }
 
 type StatisticsRequest struct {
 	Replication string `protobuf:"bytes,1,opt,name=replication" json:"replication,omitempty"`
@@ -947,7 +989,7 @@ type StatisticsRequest struct {
 func (m *StatisticsRequest) Reset()                    { *m = StatisticsRequest{} }
 func (m *StatisticsRequest) String() string            { return proto.CompactTextString(m) }
 func (*StatisticsRequest) ProtoMessage()               {}
-func (*StatisticsRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{26} }
+func (*StatisticsRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{28} }
 
 func (m *StatisticsRequest) GetReplication() string {
 	if m != nil {
@@ -982,7 +1024,7 @@ type StatisticsResponse struct {
 func (m *StatisticsResponse) Reset()                    { *m = StatisticsResponse{} }
 func (m *StatisticsResponse) String() string            { return proto.CompactTextString(m) }
 func (*StatisticsResponse) ProtoMessage()               {}
-func (*StatisticsResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{27} }
+func (*StatisticsResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{29} }
 
 func (m *StatisticsResponse) GetReplication() string {
 	if m != nil {
@@ -1032,7 +1074,7 @@ type GetFilerConfigurationRequest struct {
 func (m *GetFilerConfigurationRequest) Reset()                    { *m = GetFilerConfigurationRequest{} }
 func (m *GetFilerConfigurationRequest) String() string            { return proto.CompactTextString(m) }
 func (*GetFilerConfigurationRequest) ProtoMessage()               {}
-func (*GetFilerConfigurationRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{28} }
+func (*GetFilerConfigurationRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{30} }
 
 type GetFilerConfigurationResponse struct {
 	Masters     []string `protobuf:"bytes,1,rep,name=masters" json:"masters,omitempty"`
@@ -1046,7 +1088,7 @@ type GetFilerConfigurationResponse struct {
 func (m *GetFilerConfigurationResponse) Reset()                    { *m = GetFilerConfigurationResponse{} }
 func (m *GetFilerConfigurationResponse) String() string            { return proto.CompactTextString(m) }
 func (*GetFilerConfigurationResponse) ProtoMessage()               {}
-func (*GetFilerConfigurationResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{29} }
+func (*GetFilerConfigurationResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{31} }
 
 func (m *GetFilerConfigurationResponse) GetMasters() []string {
 	if m != nil {
@@ -1099,7 +1141,7 @@ type SubscribeMetadataRequest struct {
 func (m *SubscribeMetadataRequest) Reset()                    { *m = SubscribeMetadataRequest{} }
 func (m *SubscribeMetadataRequest) String() string            { return proto.CompactTextString(m) }
 func (*SubscribeMetadataRequest) ProtoMessage()               {}
-func (*SubscribeMetadataRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{30} }
+func (*SubscribeMetadataRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{32} }
 
 func (m *SubscribeMetadataRequest) GetClientName() string {
 	if m != nil {
@@ -1130,7 +1172,7 @@ type SubscribeMetadataResponse struct {
 func (m *SubscribeMetadataResponse) Reset()                    { *m = SubscribeMetadataResponse{} }
 func (m *SubscribeMetadataResponse) String() string            { return proto.CompactTextString(m) }
 func (*SubscribeMetadataResponse) ProtoMessage()               {}
-func (*SubscribeMetadataResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{31} }
+func (*SubscribeMetadataResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{33} }
 
 func (m *SubscribeMetadataResponse) GetDirectory() string {
 	if m != nil {
@@ -1155,7 +1197,7 @@ type LogEntry struct {
 func (m *LogEntry) Reset()                    { *m = LogEntry{} }
 func (m *LogEntry) String() string            { return proto.CompactTextString(m) }
 func (*LogEntry) ProtoMessage()               {}
-func (*LogEntry) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{32} }
+func (*LogEntry) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{34} }
 
 func (m *LogEntry) GetTsNs() int64 {
 	if m != nil {
@@ -1193,6 +1235,8 @@ func init() {
 	proto.RegisterType((*CreateEntryResponse)(nil), "filer_pb.CreateEntryResponse")
 	proto.RegisterType((*UpdateEntryRequest)(nil), "filer_pb.UpdateEntryRequest")
 	proto.RegisterType((*UpdateEntryResponse)(nil), "filer_pb.UpdateEntryResponse")
+	proto.RegisterType((*AppendToEntryRequest)(nil), "filer_pb.AppendToEntryRequest")
+	proto.RegisterType((*AppendToEntryResponse)(nil), "filer_pb.AppendToEntryResponse")
 	proto.RegisterType((*DeleteEntryRequest)(nil), "filer_pb.DeleteEntryRequest")
 	proto.RegisterType((*DeleteEntryResponse)(nil), "filer_pb.DeleteEntryResponse")
 	proto.RegisterType((*AtomicRenameEntryRequest)(nil), "filer_pb.AtomicRenameEntryRequest")
@@ -1229,6 +1273,7 @@ type SeaweedFilerClient interface {
 	ListEntries(ctx context.Context, in *ListEntriesRequest, opts ...grpc.CallOption) (SeaweedFiler_ListEntriesClient, error)
 	CreateEntry(ctx context.Context, in *CreateEntryRequest, opts ...grpc.CallOption) (*CreateEntryResponse, error)
 	UpdateEntry(ctx context.Context, in *UpdateEntryRequest, opts ...grpc.CallOption) (*UpdateEntryResponse, error)
+	AppendToEntry(ctx context.Context, in *AppendToEntryRequest, opts ...grpc.CallOption) (*AppendToEntryResponse, error)
 	DeleteEntry(ctx context.Context, in *DeleteEntryRequest, opts ...grpc.CallOption) (*DeleteEntryResponse, error)
 	AtomicRenameEntry(ctx context.Context, in *AtomicRenameEntryRequest, opts ...grpc.CallOption) (*AtomicRenameEntryResponse, error)
 	AssignVolume(ctx context.Context, in *AssignVolumeRequest, opts ...grpc.CallOption) (*AssignVolumeResponse, error)
@@ -1306,6 +1351,15 @@ func (c *seaweedFilerClient) UpdateEntry(ctx context.Context, in *UpdateEntryReq
 	return out, nil
 }
 
+func (c *seaweedFilerClient) AppendToEntry(ctx context.Context, in *AppendToEntryRequest, opts ...grpc.CallOption) (*AppendToEntryResponse, error) {
+	out := new(AppendToEntryResponse)
+	err := grpc.Invoke(ctx, "/filer_pb.SeaweedFiler/AppendToEntry", in, out, c.cc, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
 func (c *seaweedFilerClient) DeleteEntry(ctx context.Context, in *DeleteEntryRequest, opts ...grpc.CallOption) (*DeleteEntryResponse, error) {
 	out := new(DeleteEntryResponse)
 	err := grpc.Invoke(ctx, "/filer_pb.SeaweedFiler/DeleteEntry", in, out, c.cc, opts...)
@@ -1408,6 +1462,7 @@ type SeaweedFilerServer interface {
 	ListEntries(*ListEntriesRequest, SeaweedFiler_ListEntriesServer) error
 	CreateEntry(context.Context, *CreateEntryRequest) (*CreateEntryResponse, error)
 	UpdateEntry(context.Context, *UpdateEntryRequest) (*UpdateEntryResponse, error)
+	AppendToEntry(context.Context, *AppendToEntryRequest) (*AppendToEntryResponse, error)
 	DeleteEntry(context.Context, *DeleteEntryRequest) (*DeleteEntryResponse, error)
 	AtomicRenameEntry(context.Context, *AtomicRenameEntryRequest) (*AtomicRenameEntryResponse, error)
 	AssignVolume(context.Context, *AssignVolumeRequest) (*AssignVolumeResponse, error)
@@ -1497,6 +1552,24 @@ func _SeaweedFiler_UpdateEntry_Handler(srv interface{}, ctx context.Context, dec
 	return interceptor(ctx, in, info, handler)
 }
 
+func _SeaweedFiler_AppendToEntry_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(AppendToEntryRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(SeaweedFilerServer).AppendToEntry(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: "/filer_pb.SeaweedFiler/AppendToEntry",
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(SeaweedFilerServer).AppendToEntry(ctx, req.(*AppendToEntryRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
 func _SeaweedFiler_DeleteEntry_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
 	in := new(DeleteEntryRequest)
 	if err := dec(in); err != nil {
@@ -1660,6 +1733,10 @@ var _SeaweedFiler_serviceDesc = grpc.ServiceDesc{
 			MethodName: "UpdateEntry",
 			Handler:    _SeaweedFiler_UpdateEntry_Handler,
 		},
+		{
+			MethodName: "AppendToEntry",
+			Handler:    _SeaweedFiler_AppendToEntry_Handler,
+		},
 		{
 			MethodName: "DeleteEntry",
 			Handler:    _SeaweedFiler_DeleteEntry_Handler,
@@ -1707,124 +1784,128 @@ var _SeaweedFiler_serviceDesc = grpc.ServiceDesc{
 func init() { proto.RegisterFile("filer.proto", fileDescriptor0) }
 
 var fileDescriptor0 = []byte{
-	// 1903 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xb4, 0x58, 0x5f, 0x6f, 0xdc, 0xc6,
-	0x11, 0x37, 0xef, 0x3f, 0xe7, 0xee, 0x1c, 0x69, 0x4f, 0x4e, 0x4e, 0x67, 0xc9, 0x51, 0xe8, 0x3a,
-	0x75, 0x61, 0x43, 0x35, 0xd4, 0x14, 0x48, 0x9a, 0xf6, 0xc1, 0x96, 0xe5, 0xd4, 0x8d, 0xad, 0x18,
-	0x94, 0x5d, 0xb4, 0x28, 0x50, 0x96, 0x47, 0xae, 0xee, 0xb6, 0xe2, 0x91, 0xec, 0xee, 0x52, 0x7f,
-	0xf2, 0xd4, 0x97, 0x7e, 0x89, 0x02, 0xfd, 0x00, 0x7d, 0xef, 0x63, 0xd1, 0x97, 0xa2, 0x40, 0x81,
-	0x7e, 0x8b, 0x7e, 0x92, 0x62, 0x67, 0x49, 0xde, 0xf2, 0xfe, 0x48, 0x09, 0x8a, 0xbc, 0xed, 0xce,
-	0xcc, 0xce, 0xce, 0xce, 0x9f, 0xdf, 0x0c, 0x09, 0xdd, 0x53, 0x16, 0x51, 0xbe, 0x9f, 0xf2, 0x44,
-	0x26, 0xa4, 0x83, 0x1b, 0x2f, 0x1d, 0x3b, 0x5f, 0xc1, 0xdd, 0x57, 0x49, 0x72, 0x96, 0xa5, 0xcf,
-	0x19, 0xa7, 0x81, 0x4c, 0xf8, 0xd5, 0x51, 0x2c, 0xf9, 0x95, 0x4b, 0xff, 0x90, 0x51, 0x21, 0xc9,
-	0x0e, 0xd8, 0x61, 0xc1, 0x18, 0x5a, 0x7b, 0xd6, 0x43, 0xdb, 0x9d, 0x13, 0x08, 0x81, 0x46, 0xec,
-	0xcf, 0xe8, 0xb0, 0x86, 0x0c, 0x5c, 0x3b, 0x47, 0xb0, 0xb3, 0x5a, 0xa1, 0x48, 0x93, 0x58, 0x50,
-	0xf2, 0x00, 0x9a, 0x54, 0x11, 0x50, 0x5b, 0xf7, 0xe0, 0xbd, 0xfd, 0xc2, 0x94, 0x7d, 0x2d, 0xa7,
-	0xb9, 0xce, 0x3f, 0x2c, 0x20, 0xaf, 0x98, 0x90, 0x8a, 0xc8, 0xa8, 0xf8, 0x66, 0xf6, 0xbc, 0x0f,
-	0xad, 0x94, 0xd3, 0x53, 0x76, 0x99, 0x5b, 0x94, 0xef, 0xc8, 0x63, 0xd8, 0x14, 0xd2, 0xe7, 0xf2,
-	0x05, 0x4f, 0x66, 0x2f, 0x58, 0x44, 0x8f, 0x95, 0xd1, 0x75, 0x14, 0x59, 0x66, 0x90, 0x7d, 0x20,
-	0x2c, 0x0e, 0xa2, 0x4c, 0xb0, 0x73, 0x7a, 0x52, 0x70, 0x87, 0x8d, 0x3d, 0xeb, 0x61, 0xc7, 0x5d,
-	0xc1, 0x21, 0x5b, 0xd0, 0x8c, 0xd8, 0x8c, 0xc9, 0x61, 0x73, 0xcf, 0x7a, 0xd8, 0x77, 0xf5, 0xc6,
-	0xf9, 0x29, 0x0c, 0x2a, 0xf6, 0x7f, 0xbb, 0xe7, 0xff, 0xa5, 0x06, 0x4d, 0x24, 0x94, 0x3e, 0xb6,
-	0xe6, 0x3e, 0x26, 0x1f, 0x41, 0x8f, 0x09, 0x6f, 0xee, 0x88, 0x1a, 0xda, 0xd6, 0x65, 0xa2, 0xf4,
-	0x39, 0x79, 0x04, 0xad, 0x60, 0x9a, 0xc5, 0x67, 0x62, 0x58, 0xdf, 0xab, 0x3f, 0xec, 0x1e, 0x0c,
-	0xe6, 0x17, 0xa9, 0x87, 0x1e, 0x2a, 0x9e, 0x9b, 0x8b, 0x90, 0x4f, 0x01, 0x7c, 0x29, 0x39, 0x1b,
-	0x67, 0x92, 0x0a, 0x7c, 0x69, 0xf7, 0x60, 0x68, 0x1c, 0xc8, 0x04, 0x7d, 0x5a, 0xf2, 0x5d, 0x43,
-	0x96, 0x7c, 0x06, 0x1d, 0x7a, 0x29, 0x69, 0x1c, 0xd2, 0x70, 0xd8, 0xc4, 0x8b, 0x76, 0x17, 0x5e,
-	0xb4, 0x7f, 0x94, 0xf3, 0xf5, 0xfb, 0x4a, 0xf1, 0xd1, 0xe7, 0xd0, 0xaf, 0xb0, 0xc8, 0x06, 0xd4,
-	0xcf, 0x68, 0x11, 0x55, 0xb5, 0x54, 0x9e, 0x3d, 0xf7, 0xa3, 0x4c, 0x27, 0x58, 0xcf, 0xd5, 0x9b,
-	0x9f, 0xd4, 0x3e, 0xb5, 0x9c, 0xe7, 0x60, 0xbf, 0xc8, 0xa2, 0xa8, 0x3c, 0x18, 0x32, 0x5e, 0x1c,
-	0x0c, 0x19, 0x9f, 0x7b, 0xb9, 0x76, 0xad, 0x97, 0xff, 0x6e, 0xc1, 0xe6, 0xd1, 0x39, 0x8d, 0xe5,
-	0x71, 0x22, 0xd9, 0x29, 0x0b, 0x7c, 0xc9, 0x92, 0x98, 0x3c, 0x06, 0x3b, 0x89, 0x42, 0xef, 0xda,
-	0x30, 0x75, 0x92, 0x28, 0xb7, 0xfa, 0x31, 0xd8, 0x31, 0xbd, 0xf0, 0xae, 0xbd, 0xae, 0x13, 0xd3,
-	0x0b, 0x2d, 0x7d, 0x1f, 0xfa, 0x21, 0x8d, 0xa8, 0xa4, 0x5e, 0x19, 0x1d, 0x15, 0xba, 0x9e, 0x26,
-	0x1e, 0xea, 0x70, 0x7c, 0x0c, 0xef, 0x29, 0x95, 0xa9, 0xcf, 0x69, 0x2c, 0xbd, 0xd4, 0x97, 0x53,
-	0x8c, 0x89, 0xed, 0xf6, 0x63, 0x7a, 0xf1, 0x06, 0xa9, 0x6f, 0x7c, 0x39, 0x75, 0xfe, 0x56, 0x03,
-	0xbb, 0x0c, 0x26, 0xf9, 0x00, 0xda, 0xea, 0x5a, 0x8f, 0x85, 0xb9, 0x27, 0x5a, 0x6a, 0xfb, 0x32,
-	0x54, 0x55, 0x91, 0x9c, 0x9e, 0x0a, 0x2a, 0xd1, 0xbc, 0xba, 0x9b, 0xef, 0x54, 0x66, 0x09, 0xf6,
-	0xb5, 0x2e, 0x84, 0x86, 0x8b, 0x6b, 0xe5, 0xf1, 0x99, 0x64, 0x33, 0x8a, 0x17, 0xd6, 0x5d, 0xbd,
-	0x21, 0x03, 0x68, 0x52, 0x4f, 0xfa, 0x13, 0xcc, 0x70, 0xdb, 0x6d, 0xd0, 0xb7, 0xfe, 0x84, 0x7c,
-	0x0f, 0x6e, 0x8b, 0x24, 0xe3, 0x01, 0xf5, 0x8a, 0x6b, 0x5b, 0xc8, 0xed, 0x69, 0xea, 0x0b, 0x7d,
-	0xb9, 0x03, 0xf5, 0x53, 0x16, 0x0e, 0xdb, 0xe8, 0x98, 0x8d, 0x6a, 0x12, 0xbe, 0x0c, 0x5d, 0xc5,
-	0x24, 0x3f, 0x04, 0x28, 0x35, 0x85, 0xc3, 0xce, 0x1a, 0x51, 0xbb, 0xd0, 0x1b, 0x92, 0x5d, 0x80,
-	0x80, 0xa5, 0x53, 0xca, 0x3d, 0x95, 0x30, 0x36, 0x26, 0x87, 0xad, 0x29, 0x5f, 0xd2, 0x2b, 0xc5,
-	0x66, 0xc2, 0x9b, 0x7c, 0xcd, 0xd2, 0x94, 0x86, 0x43, 0x40, 0x0f, 0xdb, 0x4c, 0x7c, 0xa1, 0x09,
-	0xce, 0xaf, 0xa0, 0x95, 0x1b, 0x77, 0x17, 0xec, 0xf3, 0x24, 0xca, 0x66, 0xa5, 0xd3, 0xfa, 0x6e,
-	0x47, 0x13, 0x5e, 0x86, 0x64, 0x1b, 0x10, 0x25, 0xf1, 0x8a, 0x1a, 0xba, 0x08, 0xfd, 0xab, 0x2e,
-	0x78, 0x1f, 0x5a, 0x41, 0x92, 0x9c, 0x31, 0xed, 0xbb, 0xb6, 0x9b, 0xef, 0x9c, 0x3f, 0xd6, 0xe1,
-	0x76, 0xb5, 0x58, 0xd4, 0x15, 0xa8, 0x05, 0x3d, 0x6d, 0xa1, 0x1a, 0x54, 0x7b, 0x52, 0xf1, 0x76,
-	0xcd, 0xf4, 0x76, 0x71, 0x64, 0x96, 0x84, 0xfa, 0x82, 0xbe, 0x3e, 0xf2, 0x3a, 0x09, 0xa9, 0xca,
-	0xf5, 0x8c, 0x85, 0x18, 0x9e, 0xbe, 0xab, 0x96, 0x8a, 0x32, 0x61, 0x61, 0x0e, 0x3e, 0x6a, 0x89,
-	0xe6, 0x71, 0xd4, 0xdb, 0xd2, 0x01, 0xd7, 0x3b, 0x15, 0xf0, 0x99, 0xa2, 0xb6, 0x75, 0x14, 0xd5,
-	0x9a, 0xec, 0x41, 0x97, 0xd3, 0x34, 0xca, 0x73, 0x1f, 0x9d, 0x6f, 0xbb, 0x26, 0x89, 0xdc, 0x03,
-	0x08, 0x92, 0x28, 0xa2, 0x01, 0x0a, 0xd8, 0x28, 0x60, 0x50, 0x54, 0xde, 0x49, 0x19, 0x79, 0x82,
-	0x06, 0xe8, 0xea, 0xa6, 0xdb, 0x92, 0x32, 0x3a, 0xa1, 0x81, 0x7a, 0x47, 0x26, 0x28, 0xf7, 0x10,
-	0xbe, 0xba, 0x78, 0xae, 0xa3, 0x08, 0x08, 0xb2, 0xbb, 0x00, 0x13, 0x9e, 0x64, 0xa9, 0xe6, 0xf6,
-	0xf6, 0xea, 0x0a, 0xc9, 0x91, 0x82, 0xec, 0x07, 0x70, 0x5b, 0x5c, 0xcd, 0x22, 0x16, 0x9f, 0x79,
-	0xd2, 0xe7, 0x13, 0x2a, 0x87, 0x7d, 0x5d, 0x01, 0x39, 0xf5, 0x2d, 0x12, 0xd5, 0xdb, 0x67, 0xe1,
-	0x8f, 0x87, 0xb7, 0x31, 0x03, 0xd4, 0xd2, 0x49, 0x81, 0x1c, 0x72, 0xea, 0x4b, 0xfa, 0x2d, 0xda,
-	0xd8, 0x37, 0x43, 0x0b, 0x72, 0x07, 0x5a, 0x89, 0x47, 0x2f, 0x83, 0x28, 0x2f, 0xda, 0x66, 0x72,
-	0x74, 0x19, 0x44, 0xce, 0x23, 0x18, 0x54, 0x6e, 0xcc, 0x81, 0x7e, 0x0b, 0x9a, 0x94, 0xf3, 0xa4,
-	0x80, 0x25, 0xbd, 0x71, 0x7e, 0x0d, 0xe4, 0x5d, 0x1a, 0x7e, 0x17, 0xe6, 0x39, 0x77, 0x60, 0x50,
-	0x51, 0xad, 0xed, 0x70, 0xfe, 0x65, 0x01, 0x79, 0x8e, 0xe8, 0xf2, 0xff, 0x35, 0x76, 0x55, 0xef,
-	0xaa, 0xe9, 0x68, 0xf4, 0x0a, 0x7d, 0xe9, 0xe7, 0x2d, 0xb1, 0xc7, 0x84, 0xd6, 0xff, 0xdc, 0x97,
-	0x7e, 0xde, 0x9a, 0x38, 0x0d, 0x32, 0xae, 0xba, 0x24, 0xa6, 0x25, 0xb6, 0x26, 0xb7, 0x20, 0x91,
-	0x4f, 0xe0, 0x7d, 0x36, 0x89, 0x13, 0x4e, 0xe7, 0x62, 0x9e, 0x76, 0x55, 0x0b, 0x85, 0xb7, 0x34,
-	0xb7, 0x3c, 0x70, 0x84, 0x9e, 0x7b, 0x04, 0x83, 0xca, 0x33, 0xae, 0x75, 0xf3, 0x9f, 0x2d, 0x18,
-	0x3e, 0x95, 0xc9, 0x8c, 0x05, 0x2e, 0x55, 0xc6, 0x57, 0x9e, 0x7e, 0x1f, 0xfa, 0x0a, 0xdf, 0x17,
-	0x9f, 0xdf, 0x4b, 0xa2, 0x70, 0xde, 0x3f, 0xb7, 0x41, 0x41, 0xbc, 0x67, 0x78, 0xa1, 0x9d, 0x44,
-	0x21, 0xe6, 0xe6, 0x7d, 0x50, 0x38, 0x6c, 0x9c, 0xd7, 0x93, 0x44, 0x2f, 0xa6, 0x17, 0x95, 0xf3,
-	0x4a, 0x08, 0xcf, 0x6b, 0xf0, 0x6e, 0xc7, 0xf4, 0x42, 0x9d, 0x77, 0xee, 0xc2, 0xf6, 0x0a, 0xdb,
-	0xf2, 0x70, 0xfd, 0xdb, 0x82, 0xc1, 0x53, 0x21, 0xd8, 0x24, 0xfe, 0x25, 0x02, 0x51, 0x61, 0xf4,
-	0x16, 0x34, 0x83, 0x24, 0x8b, 0x25, 0x1a, 0xdb, 0x74, 0xf5, 0x66, 0xa1, 0x36, 0x6b, 0x4b, 0xb5,
-	0xb9, 0x50, 0xdd, 0xf5, 0xe5, 0xea, 0x36, 0xaa, 0xb7, 0x51, 0xa9, 0xde, 0x0f, 0xa1, 0xab, 0x82,
-	0xec, 0x05, 0x34, 0x96, 0x94, 0xe7, 0xc8, 0x0f, 0x8a, 0x74, 0x88, 0x14, 0x25, 0x60, 0x76, 0x28,
-	0x0d, 0xfe, 0x90, 0xce, 0xdb, 0xd3, 0x7f, 0x2d, 0xd8, 0xaa, 0x3e, 0x25, 0x8f, 0xd9, 0xda, 0x4e,
-	0xa5, 0xc0, 0x8d, 0x47, 0xf9, 0x3b, 0xd4, 0x52, 0xc1, 0x44, 0x9a, 0x8d, 0x23, 0x16, 0x78, 0x8a,
-	0xa1, 0xed, 0xb7, 0x35, 0xe5, 0x1d, 0x8f, 0xe6, 0x5e, 0x69, 0x98, 0x5e, 0x21, 0xd0, 0xf0, 0x33,
-	0x39, 0x2d, 0xba, 0x95, 0x5a, 0x2f, 0x78, 0xaa, 0x75, 0x93, 0xa7, 0xda, 0xcb, 0x9e, 0x2a, 0x33,
-	0xad, 0x63, 0x66, 0xda, 0x27, 0x30, 0xd0, 0xe3, 0x6e, 0x35, 0x5c, 0xbb, 0x00, 0x65, 0x67, 0x11,
-	0x43, 0x4b, 0xc3, 0x5b, 0xd1, 0x5a, 0x84, 0xf3, 0x33, 0xb0, 0x5f, 0x25, 0x5a, 0xaf, 0x20, 0x4f,
-	0xc0, 0x8e, 0x8a, 0x0d, 0x8a, 0x76, 0x0f, 0xc8, 0xbc, 0xc6, 0x0b, 0x39, 0x77, 0x2e, 0xe4, 0x7c,
-	0x0e, 0x9d, 0x82, 0x5c, 0xf8, 0xcc, 0x5a, 0xe7, 0xb3, 0xda, 0x82, 0xcf, 0x9c, 0x7f, 0x5a, 0xb0,
-	0x55, 0x35, 0x39, 0x0f, 0xcb, 0x3b, 0xe8, 0x97, 0x57, 0x78, 0x33, 0x3f, 0xcd, 0x6d, 0x79, 0x62,
-	0xda, 0xb2, 0x7c, 0xac, 0x34, 0x50, 0xbc, 0xf6, 0x53, 0x9d, 0xcb, 0xbd, 0xc8, 0x20, 0x8d, 0xde,
-	0xc2, 0xe6, 0x92, 0xc8, 0x8a, 0x59, 0xef, 0x07, 0xe6, 0xac, 0x57, 0x99, 0x57, 0xcb, 0xd3, 0xe6,
-	0x00, 0xf8, 0x19, 0x7c, 0xa0, 0xe1, 0xe0, 0xb0, 0x8c, 0x61, 0xe1, 0xfb, 0x6a, 0xa8, 0xad, 0xc5,
-	0x50, 0x3b, 0x23, 0x18, 0x2e, 0x1f, 0xcd, 0xcb, 0x6f, 0x02, 0x9b, 0x27, 0xd2, 0x97, 0x4c, 0x48,
-	0x16, 0x94, 0x1f, 0x1d, 0x0b, 0xb9, 0x61, 0xdd, 0xd4, 0x23, 0x97, 0xeb, 0x70, 0x03, 0xea, 0x52,
-	0x16, 0xf9, 0xab, 0x96, 0x2a, 0x0a, 0xc4, 0xbc, 0x29, 0x8f, 0xc1, 0x77, 0x70, 0x95, 0xca, 0x07,
-	0x99, 0x48, 0x3f, 0xd2, 0x33, 0x48, 0x03, 0x67, 0x10, 0x1b, 0x29, 0x38, 0x84, 0xe8, 0x36, 0x1d,
-	0x6a, 0x6e, 0x53, 0x4f, 0x28, 0x8a, 0x80, 0xcc, 0x5d, 0x00, 0x2c, 0x55, 0x5d, 0x65, 0x2d, 0x7d,
-	0x56, 0x51, 0x0e, 0x15, 0xc1, 0xb9, 0x07, 0x3b, 0x5f, 0x50, 0xa9, 0xa6, 0x29, 0x7e, 0x98, 0xc4,
-	0xa7, 0x6c, 0x92, 0x71, 0xdf, 0x08, 0x85, 0xf3, 0x1f, 0x0b, 0x76, 0xd7, 0x08, 0xe4, 0x0f, 0x1e,
-	0x42, 0x7b, 0xe6, 0x0b, 0x49, 0x79, 0x51, 0x25, 0xc5, 0x76, 0xd1, 0x15, 0xb5, 0x9b, 0x5c, 0x51,
-	0x5f, 0x72, 0xc5, 0x1d, 0x68, 0xcd, 0xfc, 0x4b, 0x6f, 0x36, 0xce, 0xc7, 0xa5, 0xe6, 0xcc, 0xbf,
-	0x7c, 0x3d, 0x46, 0x64, 0x63, 0xdc, 0x1b, 0x67, 0xc1, 0x19, 0x95, 0xa2, 0x44, 0x36, 0xc6, 0x9f,
-	0x69, 0x0a, 0xce, 0x4f, 0x38, 0x4c, 0x22, 0x0c, 0x74, 0xdc, 0x7c, 0xe7, 0x5c, 0xc0, 0xf0, 0x24,
-	0x1b, 0x8b, 0x80, 0xb3, 0x31, 0x7d, 0x4d, 0xa5, 0xaf, 0xc0, 0xb0, 0xc8, 0x91, 0x0f, 0xa1, 0x1b,
-	0x44, 0x4c, 0xa1, 0xa1, 0xf1, 0xb5, 0x06, 0x9a, 0x84, 0x5d, 0x03, 0xe1, 0x52, 0x4e, 0xbd, 0xca,
-	0x07, 0x2a, 0x28, 0xd2, 0x1b, 0xfd, 0x91, 0xba, 0x0d, 0x1d, 0xc1, 0xe2, 0x80, 0x7a, 0xb1, 0xfe,
-	0x2a, 0xa8, 0xbb, 0x6d, 0xdc, 0x1f, 0x0b, 0xe7, 0x4f, 0x16, 0x6c, 0xaf, 0xb8, 0x39, 0x77, 0xe1,
-	0xf5, 0xad, 0xfc, 0x17, 0x40, 0xe8, 0x39, 0xda, 0x65, 0x7c, 0xe3, 0xe4, 0x45, 0x76, 0xd7, 0x18,
-	0x25, 0x16, 0x3f, 0x83, 0xdc, 0x4d, 0xba, 0x48, 0x72, 0x7c, 0x85, 0x3b, 0x13, 0x5d, 0xc1, 0x03,
-	0x68, 0x4a, 0xe1, 0x21, 0x62, 0x29, 0x5b, 0x1b, 0x52, 0x1c, 0x0b, 0xf2, 0x18, 0x48, 0xea, 0x73,
-	0xc9, 0x94, 0xb4, 0x1a, 0x9c, 0xbd, 0xa9, 0x2f, 0xa6, 0x78, 0x59, 0xd3, 0xdd, 0x28, 0x39, 0x5f,
-	0xd2, 0xab, 0x9f, 0xfb, 0x62, 0xaa, 0x70, 0x1a, 0xe7, 0x88, 0x3a, 0x8e, 0x6f, 0xb8, 0x3e, 0xf8,
-	0x6b, 0x07, 0x7a, 0x27, 0xd4, 0xbf, 0xa0, 0x34, 0xc4, 0xac, 0x21, 0x93, 0x02, 0xad, 0xaa, 0xff,
-	0x13, 0xc8, 0x83, 0x45, 0x58, 0x5a, 0xf9, 0x03, 0x63, 0xf4, 0xf1, 0x4d, 0x62, 0x79, 0xe1, 0xdf,
-	0x22, 0xc7, 0xd0, 0x35, 0x3e, 0xd8, 0xc9, 0x8e, 0x71, 0x70, 0xe9, 0x3f, 0xc4, 0x68, 0x77, 0x0d,
-	0xb7, 0xd0, 0xf6, 0xc4, 0x22, 0xaf, 0xa0, 0x6b, 0xcc, 0x85, 0xa6, 0xbe, 0xe5, 0x01, 0xd5, 0xd4,
-	0xb7, 0x62, 0x98, 0x74, 0x6e, 0x29, 0x6d, 0xc6, 0x74, 0x67, 0x6a, 0x5b, 0x9e, 0x27, 0x4d, 0x6d,
-	0xab, 0x46, 0x42, 0xd4, 0x66, 0x0c, 0x53, 0xa6, 0xb6, 0xe5, 0x51, 0xd1, 0xd4, 0xb6, 0x62, 0x02,
-	0x73, 0x6e, 0x91, 0xdf, 0xc2, 0xe6, 0xd2, 0x40, 0x43, 0x9c, 0xf9, 0xa9, 0x75, 0x93, 0xd8, 0xe8,
-	0xfe, 0xb5, 0x32, 0xa5, 0xfe, 0xaf, 0xa0, 0x67, 0xce, 0x11, 0xc4, 0x30, 0x68, 0xc5, 0xa8, 0x34,
-	0xba, 0xb7, 0x8e, 0x6d, 0x2a, 0x34, 0x5b, 0x99, 0xa9, 0x70, 0x45, 0x33, 0x37, 0x15, 0xae, 0xea,
-	0x80, 0xce, 0x2d, 0xf2, 0x1b, 0xd8, 0x58, 0x6c, 0x29, 0xe4, 0xa3, 0x45, 0xb7, 0x2d, 0x75, 0xaa,
-	0x91, 0x73, 0x9d, 0x48, 0xa9, 0xfc, 0x25, 0xc0, 0xbc, 0x53, 0x10, 0xa3, 0x66, 0x97, 0x3a, 0xd5,
-	0x68, 0x67, 0x35, 0xb3, 0x54, 0xf5, 0x7b, 0xb8, 0xb3, 0x12, 0x8e, 0x89, 0x51, 0x26, 0xd7, 0x01,
-	0xfa, 0xe8, 0xfb, 0x37, 0xca, 0x95, 0x77, 0xfd, 0x0e, 0x36, 0x97, 0x30, 0xcb, 0xcc, 0x8a, 0x75,
-	0x50, 0x6a, 0x66, 0xc5, 0x5a, 0xd0, 0x53, 0x15, 0xf6, 0xec, 0x1e, 0x6c, 0x08, 0x0d, 0x15, 0xa7,
-	0x62, 0x5f, 0x43, 0xed, 0x33, 0x40, 0x9b, 0xde, 0xf0, 0x44, 0x26, 0xe3, 0x16, 0xfe, 0xec, 0xfc,
-	0xd1, 0xff, 0x02, 0x00, 0x00, 0xff, 0xff, 0x37, 0x58, 0xa5, 0x1f, 0xfb, 0x14, 0x00, 0x00,
+	// 1956 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xb4, 0x58, 0x5f, 0x6f, 0xdb, 0xc8,
+	0x11, 0x37, 0x25, 0x4b, 0x16, 0x47, 0x52, 0xce, 0x5e, 0xdb, 0x89, 0xac, 0xd8, 0x8e, 0x8f, 0x69,
+	0xae, 0x29, 0x12, 0xb8, 0x81, 0x7b, 0x05, 0xee, 0x7a, 0xed, 0x43, 0xe2, 0x38, 0xd7, 0xf4, 0x12,
+	0x5f, 0x40, 0x27, 0x45, 0x8b, 0x02, 0x65, 0x29, 0x72, 0x2d, 0x6d, 0x4d, 0x91, 0xec, 0xee, 0xd2,
+	0x7f, 0xee, 0xe9, 0x5e, 0xfa, 0x25, 0x0a, 0xf4, 0x5b, 0xf4, 0xb1, 0xe8, 0x4b, 0x51, 0xa0, 0x40,
+	0xbf, 0x45, 0xbf, 0x47, 0x81, 0x62, 0x67, 0x49, 0x6a, 0xa9, 0x3f, 0xf6, 0x05, 0x87, 0xbc, 0xed,
+	0xce, 0xcc, 0xce, 0xce, 0xce, 0x9f, 0xdf, 0x0c, 0x09, 0xed, 0x53, 0x16, 0x51, 0xbe, 0x9f, 0xf2,
+	0x44, 0x26, 0xa4, 0x85, 0x1b, 0x2f, 0x1d, 0x38, 0x5f, 0xc3, 0xdd, 0x57, 0x49, 0x72, 0x96, 0xa5,
+	0xcf, 0x19, 0xa7, 0x81, 0x4c, 0xf8, 0xd5, 0x51, 0x2c, 0xf9, 0x95, 0x4b, 0xff, 0x94, 0x51, 0x21,
+	0xc9, 0x36, 0xd8, 0x61, 0xc1, 0xe8, 0x59, 0x7b, 0xd6, 0x43, 0xdb, 0x9d, 0x10, 0x08, 0x81, 0xe5,
+	0xd8, 0x1f, 0xd3, 0x5e, 0x0d, 0x19, 0xb8, 0x76, 0x8e, 0x60, 0x7b, 0xbe, 0x42, 0x91, 0x26, 0xb1,
+	0xa0, 0xe4, 0x01, 0x34, 0xa8, 0x22, 0xa0, 0xb6, 0xf6, 0xc1, 0x47, 0xfb, 0x85, 0x29, 0xfb, 0x5a,
+	0x4e, 0x73, 0x9d, 0x7f, 0x58, 0x40, 0x5e, 0x31, 0x21, 0x15, 0x91, 0x51, 0xf1, 0xdd, 0xec, 0xb9,
+	0x0d, 0xcd, 0x94, 0xd3, 0x53, 0x76, 0x99, 0x5b, 0x94, 0xef, 0xc8, 0x63, 0x58, 0x13, 0xd2, 0xe7,
+	0xf2, 0x05, 0x4f, 0xc6, 0x2f, 0x58, 0x44, 0x8f, 0x95, 0xd1, 0x75, 0x14, 0x99, 0x65, 0x90, 0x7d,
+	0x20, 0x2c, 0x0e, 0xa2, 0x4c, 0xb0, 0x73, 0x7a, 0x52, 0x70, 0x7b, 0xcb, 0x7b, 0xd6, 0xc3, 0x96,
+	0x3b, 0x87, 0x43, 0x36, 0xa0, 0x11, 0xb1, 0x31, 0x93, 0xbd, 0xc6, 0x9e, 0xf5, 0xb0, 0xeb, 0xea,
+	0x8d, 0xf3, 0x73, 0x58, 0xaf, 0xd8, 0xff, 0x7e, 0xcf, 0xff, 0x6b, 0x0d, 0x1a, 0x48, 0x28, 0x7d,
+	0x6c, 0x4d, 0x7c, 0x4c, 0x3e, 0x86, 0x0e, 0x13, 0xde, 0xc4, 0x11, 0x35, 0xb4, 0xad, 0xcd, 0x44,
+	0xe9, 0x73, 0xf2, 0x08, 0x9a, 0xc1, 0x28, 0x8b, 0xcf, 0x44, 0xaf, 0xbe, 0x57, 0x7f, 0xd8, 0x3e,
+	0x58, 0x9f, 0x5c, 0xa4, 0x1e, 0x7a, 0xa8, 0x78, 0x6e, 0x2e, 0x42, 0x3e, 0x03, 0xf0, 0xa5, 0xe4,
+	0x6c, 0x90, 0x49, 0x2a, 0xf0, 0xa5, 0xed, 0x83, 0x9e, 0x71, 0x20, 0x13, 0xf4, 0x69, 0xc9, 0x77,
+	0x0d, 0x59, 0xf2, 0x39, 0xb4, 0xe8, 0xa5, 0xa4, 0x71, 0x48, 0xc3, 0x5e, 0x03, 0x2f, 0xda, 0x99,
+	0x7a, 0xd1, 0xfe, 0x51, 0xce, 0xd7, 0xef, 0x2b, 0xc5, 0xfb, 0x5f, 0x40, 0xb7, 0xc2, 0x22, 0xab,
+	0x50, 0x3f, 0xa3, 0x45, 0x54, 0xd5, 0x52, 0x79, 0xf6, 0xdc, 0x8f, 0x32, 0x9d, 0x60, 0x1d, 0x57,
+	0x6f, 0x7e, 0x56, 0xfb, 0xcc, 0x72, 0x9e, 0x83, 0xfd, 0x22, 0x8b, 0xa2, 0xf2, 0x60, 0xc8, 0x78,
+	0x71, 0x30, 0x64, 0x7c, 0xe2, 0xe5, 0xda, 0xb5, 0x5e, 0xfe, 0xbb, 0x05, 0x6b, 0x47, 0xe7, 0x34,
+	0x96, 0xc7, 0x89, 0x64, 0xa7, 0x2c, 0xf0, 0x25, 0x4b, 0x62, 0xf2, 0x18, 0xec, 0x24, 0x0a, 0xbd,
+	0x6b, 0xc3, 0xd4, 0x4a, 0xa2, 0xdc, 0xea, 0xc7, 0x60, 0xc7, 0xf4, 0xc2, 0xbb, 0xf6, 0xba, 0x56,
+	0x4c, 0x2f, 0xb4, 0xf4, 0x7d, 0xe8, 0x86, 0x34, 0xa2, 0x92, 0x7a, 0x65, 0x74, 0x54, 0xe8, 0x3a,
+	0x9a, 0x78, 0xa8, 0xc3, 0xf1, 0x09, 0x7c, 0xa4, 0x54, 0xa6, 0x3e, 0xa7, 0xb1, 0xf4, 0x52, 0x5f,
+	0x8e, 0x30, 0x26, 0xb6, 0xdb, 0x8d, 0xe9, 0xc5, 0x1b, 0xa4, 0xbe, 0xf1, 0xe5, 0xc8, 0xf9, 0x5b,
+	0x0d, 0xec, 0x32, 0x98, 0xe4, 0x0e, 0xac, 0xa8, 0x6b, 0x3d, 0x16, 0xe6, 0x9e, 0x68, 0xaa, 0xed,
+	0xcb, 0x50, 0x55, 0x45, 0x72, 0x7a, 0x2a, 0xa8, 0x44, 0xf3, 0xea, 0x6e, 0xbe, 0x53, 0x99, 0x25,
+	0xd8, 0x37, 0xba, 0x10, 0x96, 0x5d, 0x5c, 0x2b, 0x8f, 0x8f, 0x25, 0x1b, 0x53, 0xbc, 0xb0, 0xee,
+	0xea, 0x0d, 0x59, 0x87, 0x06, 0xf5, 0xa4, 0x3f, 0xc4, 0x0c, 0xb7, 0xdd, 0x65, 0xfa, 0xd6, 0x1f,
+	0x92, 0x1f, 0xc0, 0x2d, 0x91, 0x64, 0x3c, 0xa0, 0x5e, 0x71, 0x6d, 0x13, 0xb9, 0x1d, 0x4d, 0x7d,
+	0xa1, 0x2f, 0x77, 0xa0, 0x7e, 0xca, 0xc2, 0xde, 0x0a, 0x3a, 0x66, 0xb5, 0x9a, 0x84, 0x2f, 0x43,
+	0x57, 0x31, 0xc9, 0x8f, 0x01, 0x4a, 0x4d, 0x61, 0xaf, 0xb5, 0x40, 0xd4, 0x2e, 0xf4, 0x86, 0x64,
+	0x07, 0x20, 0x60, 0xe9, 0x88, 0x72, 0x4f, 0x25, 0x8c, 0x8d, 0xc9, 0x61, 0x6b, 0xca, 0x57, 0xf4,
+	0x4a, 0xb1, 0x99, 0xf0, 0x86, 0xdf, 0xb0, 0x34, 0xa5, 0x61, 0x0f, 0xd0, 0xc3, 0x36, 0x13, 0x5f,
+	0x6a, 0x82, 0xf3, 0x1b, 0x68, 0xe6, 0xc6, 0xdd, 0x05, 0xfb, 0x3c, 0x89, 0xb2, 0x71, 0xe9, 0xb4,
+	0xae, 0xdb, 0xd2, 0x84, 0x97, 0x21, 0xd9, 0x02, 0x44, 0x49, 0xbc, 0xa2, 0x86, 0x2e, 0x42, 0xff,
+	0xaa, 0x0b, 0x6e, 0x43, 0x33, 0x48, 0x92, 0x33, 0xa6, 0x7d, 0xb7, 0xe2, 0xe6, 0x3b, 0xe7, 0xdb,
+	0x3a, 0xdc, 0xaa, 0x16, 0x8b, 0xba, 0x02, 0xb5, 0xa0, 0xa7, 0x2d, 0x54, 0x83, 0x6a, 0x4f, 0x2a,
+	0xde, 0xae, 0x99, 0xde, 0x2e, 0x8e, 0x8c, 0x93, 0x50, 0x5f, 0xd0, 0xd5, 0x47, 0x5e, 0x27, 0x21,
+	0x55, 0xb9, 0x9e, 0xb1, 0x10, 0xc3, 0xd3, 0x75, 0xd5, 0x52, 0x51, 0x86, 0x2c, 0xcc, 0xc1, 0x47,
+	0x2d, 0xd1, 0x3c, 0x8e, 0x7a, 0x9b, 0x3a, 0xe0, 0x7a, 0xa7, 0x02, 0x3e, 0x56, 0xd4, 0x15, 0x1d,
+	0x45, 0xb5, 0x26, 0x7b, 0xd0, 0xe6, 0x34, 0x8d, 0xf2, 0xdc, 0x47, 0xe7, 0xdb, 0xae, 0x49, 0x22,
+	0xbb, 0x00, 0x41, 0x12, 0x45, 0x34, 0x40, 0x01, 0x1b, 0x05, 0x0c, 0x8a, 0xca, 0x3b, 0x29, 0x23,
+	0x4f, 0xd0, 0x00, 0x5d, 0xdd, 0x70, 0x9b, 0x52, 0x46, 0x27, 0x34, 0x50, 0xef, 0xc8, 0x04, 0xe5,
+	0x1e, 0xc2, 0x57, 0x1b, 0xcf, 0xb5, 0x14, 0x01, 0x41, 0x76, 0x07, 0x60, 0xc8, 0x93, 0x2c, 0xd5,
+	0xdc, 0xce, 0x5e, 0x5d, 0x21, 0x39, 0x52, 0x90, 0xfd, 0x00, 0x6e, 0x89, 0xab, 0x71, 0xc4, 0xe2,
+	0x33, 0x4f, 0xfa, 0x7c, 0x48, 0x65, 0xaf, 0xab, 0x2b, 0x20, 0xa7, 0xbe, 0x45, 0xa2, 0x7a, 0xfb,
+	0x38, 0xfc, 0x69, 0xef, 0x16, 0x66, 0x80, 0x5a, 0x3a, 0x29, 0x90, 0x43, 0x4e, 0x7d, 0x49, 0xdf,
+	0xa3, 0x8d, 0x7d, 0x37, 0xb4, 0x20, 0x9b, 0xd0, 0x4c, 0x3c, 0x7a, 0x19, 0x44, 0x79, 0xd1, 0x36,
+	0x92, 0xa3, 0xcb, 0x20, 0x72, 0x1e, 0xc1, 0x7a, 0xe5, 0xc6, 0x1c, 0xe8, 0x37, 0xa0, 0x41, 0x39,
+	0x4f, 0x0a, 0x58, 0xd2, 0x1b, 0xe7, 0xb7, 0x40, 0xde, 0xa5, 0xe1, 0x87, 0x30, 0xcf, 0xd9, 0x84,
+	0xf5, 0x8a, 0x6a, 0x6d, 0x87, 0xf3, 0xad, 0x05, 0x1b, 0x4f, 0xd3, 0x94, 0xc6, 0xe1, 0xdb, 0xe4,
+	0x3d, 0x2e, 0xdd, 0x01, 0x40, 0xb5, 0x9e, 0xd1, 0xe0, 0x6d, 0xa4, 0x60, 0x7c, 0xde, 0xa7, 0xbd,
+	0x38, 0x77, 0x60, 0x73, 0xca, 0x82, 0xdc, 0xb6, 0x7f, 0x59, 0x40, 0x9e, 0x23, 0xf2, 0x7d, 0xbf,
+	0xa1, 0x43, 0x61, 0x91, 0x6a, 0x88, 0x1a, 0x59, 0x43, 0x5f, 0xfa, 0x79, 0xbb, 0xee, 0x30, 0xa1,
+	0xf5, 0x3f, 0xf7, 0xa5, 0x9f, 0xb7, 0x4d, 0x4e, 0x83, 0x8c, 0xab, 0x0e, 0x8e, 0x25, 0x83, 0x6d,
+	0xd3, 0x2d, 0x48, 0xe4, 0x53, 0xb8, 0xcd, 0x86, 0x71, 0xc2, 0xe9, 0x44, 0xcc, 0xd3, 0x61, 0x6c,
+	0xa2, 0xf0, 0x86, 0xe6, 0x96, 0x07, 0x8e, 0x30, 0xaa, 0x8f, 0x60, 0xbd, 0xf2, 0x8c, 0x6b, 0x53,
+	0xe0, 0x2f, 0x16, 0xf4, 0x9e, 0xca, 0x64, 0xcc, 0x02, 0x97, 0x2a, 0xe3, 0x2b, 0x4f, 0xbf, 0x0f,
+	0x5d, 0xd5, 0x7b, 0xa6, 0x9f, 0xdf, 0x49, 0xa2, 0x70, 0xd2, 0xdb, 0xb7, 0x40, 0xb5, 0x1f, 0x33,
+	0x32, 0x2b, 0x49, 0x14, 0x62, 0x5c, 0xee, 0x83, 0xea, 0x11, 0xc6, 0x79, 0x3d, 0xe5, 0x74, 0x62,
+	0x7a, 0x51, 0x39, 0xaf, 0x84, 0xf0, 0xbc, 0x6e, 0x2c, 0x2b, 0x31, 0xbd, 0x50, 0xe7, 0x9d, 0xbb,
+	0xb0, 0x35, 0xc7, 0xb6, 0x3c, 0x5c, 0xff, 0xb6, 0x60, 0xfd, 0xa9, 0x10, 0x6c, 0x18, 0xff, 0x1a,
+	0x41, 0xb2, 0x30, 0x7a, 0x03, 0x1a, 0x41, 0x92, 0xc5, 0x12, 0x8d, 0x6d, 0xb8, 0x7a, 0x33, 0x85,
+	0x1b, 0xb5, 0x19, 0xdc, 0x98, 0x42, 0x9e, 0xfa, 0x2c, 0xf2, 0x18, 0xc8, 0xb2, 0x5c, 0x41, 0x96,
+	0x7b, 0xd0, 0x56, 0x41, 0xf6, 0x02, 0x1a, 0x4b, 0xca, 0xf3, 0xae, 0x04, 0x8a, 0x74, 0x88, 0x14,
+	0x25, 0x60, 0x76, 0x4f, 0xdd, 0x98, 0x20, 0x9d, 0xb4, 0xce, 0xff, 0xaa, 0xaa, 0xa8, 0x3c, 0x25,
+	0x8f, 0xd9, 0xc2, 0x2e, 0xaa, 0x80, 0x97, 0x47, 0xf9, 0x3b, 0xd4, 0x52, 0x95, 0x48, 0x9a, 0x0d,
+	0x22, 0x16, 0x78, 0x8a, 0xa1, 0xed, 0xb7, 0x35, 0xe5, 0x1d, 0x8f, 0x26, 0x5e, 0x59, 0x36, 0xbd,
+	0x42, 0x60, 0xd9, 0xcf, 0xe4, 0xa8, 0xe8, 0xa4, 0x6a, 0x3d, 0xe5, 0xa9, 0xe6, 0x4d, 0x9e, 0x5a,
+	0x99, 0xf5, 0x54, 0x99, 0x69, 0x2d, 0x33, 0xd3, 0x3e, 0x85, 0x75, 0x3d, 0x8a, 0x57, 0xc3, 0xb5,
+	0x03, 0x50, 0x76, 0x3d, 0xd1, 0xb3, 0x34, 0xf4, 0x16, 0x6d, 0x4f, 0x38, 0xbf, 0x00, 0xfb, 0x55,
+	0xa2, 0xf5, 0x0a, 0xf2, 0x04, 0xec, 0xa8, 0xd8, 0xa0, 0x68, 0xfb, 0x80, 0x4c, 0x4a, 0xbd, 0x90,
+	0x73, 0x27, 0x42, 0xce, 0x17, 0xd0, 0x2a, 0xc8, 0x85, 0xcf, 0xac, 0x45, 0x3e, 0xab, 0x4d, 0xf9,
+	0xcc, 0xf9, 0xa7, 0x05, 0x1b, 0x55, 0x93, 0xf3, 0xb0, 0xbc, 0x83, 0x6e, 0x79, 0x85, 0x37, 0xf6,
+	0xd3, 0xdc, 0x96, 0x27, 0xa6, 0x2d, 0xb3, 0xc7, 0x4a, 0x03, 0xc5, 0x6b, 0x3f, 0xd5, 0xb9, 0xdc,
+	0x89, 0x0c, 0x52, 0xff, 0x2d, 0xac, 0xcd, 0x88, 0xcc, 0x99, 0x43, 0x7f, 0x64, 0xce, 0xa1, 0x15,
+	0xb0, 0x2b, 0x4f, 0x9b, 0xc3, 0xe9, 0xe7, 0x70, 0x47, 0xc3, 0xc1, 0x61, 0x19, 0xc3, 0xc2, 0xf7,
+	0xd5, 0x50, 0x5b, 0xd3, 0xa1, 0x76, 0xfa, 0xd0, 0x9b, 0x3d, 0x9a, 0x97, 0xdf, 0x10, 0xd6, 0x4e,
+	0xa4, 0x2f, 0x99, 0x90, 0x2c, 0x28, 0x3f, 0x88, 0xa6, 0x72, 0xc3, 0xba, 0xa9, 0x7f, 0xcf, 0xd6,
+	0xe1, 0x2a, 0xd4, 0xa5, 0x2c, 0xf2, 0x57, 0x2d, 0x55, 0x14, 0x88, 0x79, 0x53, 0x1e, 0x83, 0x0f,
+	0x70, 0x95, 0xca, 0x07, 0x99, 0x48, 0x3f, 0xd2, 0xf3, 0xd1, 0x32, 0xce, 0x47, 0x36, 0x52, 0x70,
+	0x40, 0xd2, 0x23, 0x44, 0xa8, 0xb9, 0x0d, 0x3d, 0x3d, 0x29, 0x02, 0x32, 0x77, 0x00, 0xb0, 0x54,
+	0x75, 0x95, 0x35, 0xf5, 0x59, 0x45, 0x39, 0x54, 0x04, 0x67, 0x17, 0xb6, 0xbf, 0xa4, 0x52, 0x75,
+	0x23, 0x7e, 0x98, 0xc4, 0xa7, 0x6c, 0x98, 0x71, 0xdf, 0x08, 0x85, 0xf3, 0x1f, 0x0b, 0x76, 0x16,
+	0x08, 0xe4, 0x0f, 0xee, 0xc1, 0xca, 0xd8, 0x17, 0x92, 0xf2, 0xa2, 0x4a, 0x8a, 0xed, 0xb4, 0x2b,
+	0x6a, 0x37, 0xb9, 0xa2, 0x3e, 0xe3, 0x8a, 0x4d, 0x68, 0x8e, 0xfd, 0x4b, 0x6f, 0x3c, 0xc8, 0x47,
+	0xb9, 0xc6, 0xd8, 0xbf, 0x7c, 0x3d, 0x40, 0x64, 0x63, 0xdc, 0x1b, 0x64, 0xc1, 0x19, 0x95, 0xa2,
+	0x44, 0x36, 0xc6, 0x9f, 0x69, 0x0a, 0xce, 0x76, 0x38, 0xe8, 0x22, 0x0c, 0xb4, 0xdc, 0x7c, 0xe7,
+	0x5c, 0x40, 0xef, 0x24, 0x1b, 0x88, 0x80, 0xb3, 0x01, 0x7d, 0x4d, 0xa5, 0xaf, 0xc0, 0xb0, 0xc8,
+	0x91, 0x7b, 0xd0, 0x0e, 0x22, 0xa6, 0xd0, 0xd0, 0xf8, 0x92, 0x04, 0x4d, 0xc2, 0xae, 0x81, 0x70,
+	0x29, 0x47, 0x5e, 0xe5, 0xe3, 0x19, 0x14, 0xe9, 0x8d, 0xfe, 0x80, 0xde, 0x82, 0x96, 0x60, 0x71,
+	0x40, 0xbd, 0x58, 0x7f, 0xb1, 0xd4, 0xdd, 0x15, 0xdc, 0x1f, 0x0b, 0xe7, 0xcf, 0x16, 0x6c, 0xcd,
+	0xb9, 0x39, 0x77, 0xe1, 0xf5, 0xad, 0xfc, 0x57, 0x40, 0xe8, 0x39, 0xda, 0x65, 0x7c, 0x7f, 0xe5,
+	0x45, 0x76, 0xd7, 0x18, 0x73, 0xa6, 0x3f, 0xd1, 0xdc, 0x35, 0x3a, 0x4d, 0x72, 0x7c, 0x85, 0x3b,
+	0x43, 0x5d, 0xc1, 0xeb, 0xd0, 0x90, 0xc2, 0x43, 0xc4, 0x52, 0xb6, 0x2e, 0x4b, 0x71, 0x2c, 0xc8,
+	0x63, 0x20, 0xa9, 0xcf, 0x25, 0x53, 0xd2, 0x6a, 0xa8, 0xf7, 0x46, 0xbe, 0x18, 0xe1, 0x65, 0x0d,
+	0x77, 0xb5, 0xe4, 0x7c, 0x45, 0xaf, 0x7e, 0xe9, 0x8b, 0x91, 0xc2, 0x69, 0x9c, 0x23, 0xea, 0x38,
+	0x5a, 0xe2, 0xfa, 0xe0, 0x7f, 0x2d, 0xe8, 0x9c, 0x50, 0xff, 0x82, 0xd2, 0x10, 0xb3, 0x86, 0x0c,
+	0x0b, 0xb4, 0xaa, 0xfe, 0xeb, 0x20, 0x0f, 0xa6, 0x61, 0x69, 0xee, 0xcf, 0x95, 0xfe, 0x27, 0x37,
+	0x89, 0xe5, 0x85, 0xbf, 0x44, 0x8e, 0xa1, 0x6d, 0xfc, 0x4c, 0x20, 0xdb, 0xc6, 0xc1, 0x99, 0x7f,
+	0x24, 0xfd, 0x9d, 0x05, 0xdc, 0x42, 0xdb, 0x13, 0x8b, 0xbc, 0x82, 0xb6, 0x31, 0xb3, 0x9a, 0xfa,
+	0x66, 0x87, 0x67, 0x53, 0xdf, 0x9c, 0x41, 0xd7, 0x59, 0x52, 0xda, 0x8c, 0xc9, 0xd3, 0xd4, 0x36,
+	0x3b, 0xeb, 0x9a, 0xda, 0xe6, 0x8d, 0xab, 0x4b, 0xc4, 0x85, 0x6e, 0x65, 0x5a, 0x24, 0xbb, 0x93,
+	0x13, 0xf3, 0x06, 0xd9, 0xfe, 0xbd, 0x85, 0x7c, 0xd3, 0x42, 0x63, 0x40, 0x33, 0x2d, 0x9c, 0x1d,
+	0x3f, 0x4d, 0x0b, 0xe7, 0x4c, 0x75, 0xce, 0x12, 0xf9, 0x3d, 0xac, 0xcd, 0x0c, 0x49, 0xc4, 0x31,
+	0xac, 0x58, 0x30, 0xdd, 0xf5, 0xef, 0x5f, 0x2b, 0x53, 0xea, 0xff, 0x1a, 0x3a, 0xe6, 0x6c, 0x42,
+	0x0c, 0x83, 0xe6, 0x8c, 0x5f, 0xfd, 0xdd, 0x45, 0x6c, 0x53, 0xa1, 0xd9, 0x1e, 0x4d, 0x85, 0x73,
+	0x06, 0x04, 0x53, 0xe1, 0xbc, 0xae, 0xea, 0x2c, 0x91, 0xdf, 0xc1, 0xea, 0x74, 0x9b, 0x22, 0x1f,
+	0x4f, 0xbb, 0x6d, 0xa6, 0xfb, 0xf5, 0x9d, 0xeb, 0x44, 0x4a, 0xe5, 0x2f, 0x01, 0x26, 0xdd, 0x87,
+	0x18, 0x38, 0x30, 0xd3, 0xfd, 0xfa, 0xdb, 0xf3, 0x99, 0xa5, 0xaa, 0x3f, 0xc2, 0xe6, 0x5c, 0x88,
+	0x27, 0x46, 0xe9, 0x5d, 0xd7, 0x24, 0xfa, 0x3f, 0xbc, 0x51, 0xae, 0xbc, 0xeb, 0x0f, 0xb0, 0x36,
+	0x83, 0x83, 0x66, 0x56, 0x2c, 0x82, 0x67, 0x33, 0x2b, 0x16, 0x02, 0xa9, 0xaa, 0xda, 0x67, 0xbb,
+	0xb0, 0x2a, 0x34, 0xfc, 0x9c, 0x8a, 0x7d, 0x0d, 0xdf, 0xcf, 0x00, 0x6d, 0x7a, 0xc3, 0x13, 0x99,
+	0x0c, 0x9a, 0xf8, 0x73, 0xf7, 0x27, 0xff, 0x0f, 0x00, 0x00, 0xff, 0xff, 0xc4, 0x22, 0xe9, 0x54,
+	0xeb, 0x15, 0x00, 0x00,
 }
diff --git a/weed/server/filer_grpc_server.go b/weed/server/filer_grpc_server.go
index 549553acc..999d14b8e 100644
--- a/weed/server/filer_grpc_server.go
+++ b/weed/server/filer_grpc_server.go
@@ -217,6 +217,38 @@ func (fs *FilerServer) UpdateEntry(ctx context.Context, req *filer_pb.UpdateEntr
 	return &filer_pb.UpdateEntryResponse{}, err
 }
 
+func (fs *FilerServer) AppendToEntry(ctx context.Context, req *filer_pb.AppendToEntryRequest) (*filer_pb.AppendToEntryResponse, error) {
+
+	fullpath := util.NewFullPath(req.Directory, req.EntryName)
+	var offset int64 = 0
+	entry, err := fs.filer.FindEntry(ctx, util.FullPath(fullpath))
+	if err == filer_pb.ErrNotFound {
+		entry = &filer2.Entry{
+			FullPath: fullpath,
+			Attr: filer2.Attr{
+				Crtime: time.Now(),
+				Mtime:  time.Now(),
+				Mode:   os.FileMode(0644),
+				Uid:    OS_UID,
+				Gid:    OS_GID,
+			},
+		}
+	} else {
+		offset = int64(filer2.TotalSize(entry.Chunks))
+	}
+
+	for _, chunk := range req.Chunks {
+		chunk.Offset = offset
+		offset += int64(chunk.Size)
+	}
+
+	entry.Chunks = append(entry.Chunks, req.Chunks...)
+
+	err = fs.filer.CreateEntry(context.Background(), entry, false)
+
+	return &filer_pb.AppendToEntryResponse{}, err
+}
+
 func (fs *FilerServer) DeleteEntry(ctx context.Context, req *filer_pb.DeleteEntryRequest) (resp *filer_pb.DeleteEntryResponse, err error) {
 	err = fs.filer.DeleteEntryMetaAndData(ctx, util.JoinPath(req.Directory, req.Name), req.IsRecursive, req.IgnoreRecursiveError, req.IsDeleteData)
 	resp = &filer_pb.DeleteEntryResponse{}

From 2a458972373856cc1b31b0b7433b2aaa000bb700 Mon Sep 17 00:00:00 2001
From: Chris Lu <chris.lu@gmail.com>
Date: Fri, 17 Apr 2020 02:29:00 -0700
Subject: [PATCH 74/81] broker: read cipher value from filer

---
 weed/command/msg_broker.go          | 13 ++++++++-----
 weed/messaging/msg_broker_server.go |  7 +++----
 2 files changed, 11 insertions(+), 9 deletions(-)

diff --git a/weed/command/msg_broker.go b/weed/command/msg_broker.go
index f77582f03..36d164800 100644
--- a/weed/command/msg_broker.go
+++ b/weed/command/msg_broker.go
@@ -22,8 +22,8 @@ var (
 )
 
 type QueueOptions struct {
-	filer      *string
-	port       *int
+	filer *string
+	port  *int
 }
 
 func init() {
@@ -59,14 +59,16 @@ func (msgBrokerOpt *QueueOptions) startQueueServer() bool {
 		return false
 	}
 
-	grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client")
+	grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.msg_broker")
+	cipher := false
 
 	for {
 		err = pb.WithGrpcFilerClient(filerGrpcAddress, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
-			_, err := client.GetFilerConfiguration(context.Background(), &filer_pb.GetFilerConfigurationRequest{})
+			resp, err := client.GetFilerConfiguration(context.Background(), &filer_pb.GetFilerConfigurationRequest{})
 			if err != nil {
 				return fmt.Errorf("get filer %s configuration: %v", filerGrpcAddress, err)
 			}
+			cipher = resp.Cipher
 			return nil
 		})
 		if err != nil {
@@ -83,7 +85,8 @@ func (msgBrokerOpt *QueueOptions) startQueueServer() bool {
 		DefaultReplication: "",
 		MaxMB:              0,
 		Port:               *msgBrokerOpt.port,
-	})
+		Cipher:             cipher,
+	}, grpcDialOption)
 
 	// start grpc listener
 	grpcL, err := util.NewListener(":"+strconv.Itoa(*msgBrokerOpt.port), 0)
diff --git a/weed/messaging/msg_broker_server.go b/weed/messaging/msg_broker_server.go
index 9174ca4cf..bc842eeea 100644
--- a/weed/messaging/msg_broker_server.go
+++ b/weed/messaging/msg_broker_server.go
@@ -10,8 +10,6 @@ import (
 	"github.com/chrislusf/seaweedfs/weed/pb"
 	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
 	"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
-	"github.com/chrislusf/seaweedfs/weed/security"
-	"github.com/chrislusf/seaweedfs/weed/util"
 )
 
 type MessageBrokerOption struct {
@@ -19,6 +17,7 @@ type MessageBrokerOption struct {
 	DefaultReplication string
 	MaxMB              int
 	Port               int
+	Cipher             bool
 }
 
 type MessageBroker struct {
@@ -26,11 +25,11 @@ type MessageBroker struct {
 	grpcDialOption grpc.DialOption
 }
 
-func NewMessageBroker(option *MessageBrokerOption) (messageBroker *MessageBroker, err error) {
+func NewMessageBroker(option *MessageBrokerOption, grpcDialOption grpc.DialOption) (messageBroker *MessageBroker, err error) {
 
 	messageBroker = &MessageBroker{
 		option:         option,
-		grpcDialOption: security.LoadClientTLS(util.GetViper(), "grpc.msg_broker"),
+		grpcDialOption: grpcDialOption,
 	}
 
 	go messageBroker.loopForEver()

From 3f3dba5a68287da6a5a8e87945c1eca2bcf925f6 Mon Sep 17 00:00:00 2001
From: Chris Lu <chris.lu@gmail.com>
Date: Fri, 17 Apr 2020 02:29:38 -0700
Subject: [PATCH 75/81] broker: append message logs

---
 weed/messaging/broker_append.go          | 116 ++++++++++++++++
 weed/messaging/msg_broker_grpc_server.go |  17 ++-
 weed/pb/messaging.proto                  |  11 +-
 weed/pb/messaging_pb/messaging.pb.go     | 165 +++++++++++++----------
 4 files changed, 230 insertions(+), 79 deletions(-)
 create mode 100644 weed/messaging/broker_append.go

diff --git a/weed/messaging/broker_append.go b/weed/messaging/broker_append.go
new file mode 100644
index 000000000..07b3755c0
--- /dev/null
+++ b/weed/messaging/broker_append.go
@@ -0,0 +1,116 @@
+package messaging
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/operation"
+	"github.com/chrislusf/seaweedfs/weed/pb"
+	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
+	"github.com/chrislusf/seaweedfs/weed/pb/messaging_pb"
+	"github.com/chrislusf/seaweedfs/weed/security"
+	"github.com/chrislusf/seaweedfs/weed/util"
+)
+
+func (broker *MessageBroker) appendToFile(targetFile string, topicConfig *messaging_pb.TopicConfiguration, data []byte) error {
+
+	assignResult, uploadResult, err2 := broker.assignAndUpload(topicConfig, data)
+	if err2 != nil {
+		return err2
+	}
+
+	dir, name := util.FullPath(targetFile).DirAndName()
+
+	chunk := &filer_pb.FileChunk{
+		FileId:    assignResult.Fid,
+		Offset:    0, // needs to be fixed during appending
+		Size:      uint64(uploadResult.Size),
+		Mtime:     time.Now().UnixNano(),
+		ETag:      uploadResult.ETag,
+		IsGzipped: uploadResult.Gzip > 0,
+	}
+
+	// append the chunk
+	if err := broker.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
+
+		request := &filer_pb.AppendToEntryRequest{
+			Directory: dir,
+			EntryName: name,
+			Chunks:    []*filer_pb.FileChunk{chunk},
+		}
+
+		_, err := client.AppendToEntry(context.Background(), request)
+		if err != nil {
+			glog.V(0).Infof("append to file %v: %v", request, err)
+			return err
+		}
+
+		return nil
+	}); err != nil {
+		return fmt.Errorf("append to file %v: %v", targetFile, err)
+	}
+
+	return nil
+}
+
+func (broker *MessageBroker) assignAndUpload(topicConfig *messaging_pb.TopicConfiguration, data []byte) (*operation.AssignResult, *operation.UploadResult, error) {
+
+	var assignResult = &operation.AssignResult{}
+	var collection, replication string
+
+	// assign a volume location
+	if err := broker.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
+
+		request := &filer_pb.AssignVolumeRequest{
+			Count:       1,
+			Replication: topicConfig.Replication,
+			Collection:  topicConfig.Collection,
+		}
+
+		resp, err := client.AssignVolume(context.Background(), request)
+		if err != nil {
+			glog.V(0).Infof("assign volume failure %v: %v", request, err)
+			return err
+		}
+		if resp.Error != "" {
+			return fmt.Errorf("assign volume failure %v: %v", request, resp.Error)
+		}
+
+		assignResult.Auth = security.EncodedJwt(resp.Auth)
+		assignResult.Fid = resp.FileId
+		assignResult.Url = resp.Url
+		assignResult.PublicUrl = resp.PublicUrl
+		assignResult.Count = uint64(resp.Count)
+
+		collection, replication = resp.Collection, resp.Replication
+
+		return nil
+	}); err != nil {
+		return nil, nil, err
+	}
+
+	// upload data
+	targetUrl := fmt.Sprintf("http://%s/%s", assignResult.Url, assignResult.Fid)
+	uploadResult, err := operation.UploadData(targetUrl, "", broker.option.Cipher, data, false, "", nil, assignResult.Auth)
+	if err != nil {
+		return nil, nil, fmt.Errorf("upload data %s: %v", targetUrl, err)
+	}
+	// println("uploaded to", targetUrl)
+	return assignResult, uploadResult, nil
+}
+
+func (broker *MessageBroker) WithFilerClient(fn func(filer_pb.SeaweedFilerClient) error) (err error) {
+
+	for _, filer := range broker.option.Filers {
+		if err = pb.WithFilerClient(filer, broker.grpcDialOption, fn); err != nil {
+			glog.V(0).Infof("fail to connect to %s: %v", filer, err)
+		} else {
+			break
+		}
+	}
+
+	return
+
+}
diff --git a/weed/messaging/msg_broker_grpc_server.go b/weed/messaging/msg_broker_grpc_server.go
index 5b93b8f62..f4b8321e2 100644
--- a/weed/messaging/msg_broker_grpc_server.go
+++ b/weed/messaging/msg_broker_grpc_server.go
@@ -32,6 +32,11 @@ func (broker *MessageBroker) Publish(stream messaging_pb.SeaweedMessaging_Publis
 
 	updatesChan := make(chan int32)
 
+	// TODO look it up
+	topicConfig := &messaging_pb.TopicConfiguration{
+
+	}
+
 	go func() {
 		for update := range updatesChan {
 			if err := stream.Send(&messaging_pb.PublishResponse{
@@ -47,18 +52,16 @@ func (broker *MessageBroker) Publish(stream messaging_pb.SeaweedMessaging_Publis
 
 	logBuffer := log_buffer.NewLogBuffer(time.Minute, func(startTime, stopTime time.Time, buf []byte) {
 
-		//targetFile :=
-		fmt.Sprintf("%s/%s/%s/%04d-%02d-%02d/%02d-%02d.part%02d",
+		targetFile := fmt.Sprintf(
+			"%s/%s/%s/%04d-%02d-%02d/%02d-%02d.part%02d",
 			filer2.TopicsDir, namespace, topic,
 			startTime.Year(), startTime.Month(), startTime.Day(), startTime.Hour(), startTime.Minute(),
 			partition,
 		)
 
-		/*
-			if err := f.appendToFile(targetFile, buf); err != nil {
-				glog.V(0).Infof("log write failed %s: %v", targetFile, err)
-			}
-		*/
+		if err := broker.appendToFile(targetFile, topicConfig, buf); err != nil {
+			glog.V(0).Infof("log write failed %s: %v", targetFile, err)
+		}
 
 	}, func() {
 		// notify subscribers
diff --git a/weed/pb/messaging.proto b/weed/pb/messaging.proto
index 10e317221..0139a57b8 100644
--- a/weed/pb/messaging.proto
+++ b/weed/pb/messaging.proto
@@ -90,8 +90,7 @@ message PublishResponse {
 message ConfigureTopicRequest {
     string namespace = 1;
     string topic = 2;
-    int32 partition_count = 3;
-    string collection = 4;
+    TopicConfiguration configuration = 3;
 }
 message ConfigureTopicResponse {
 }
@@ -101,5 +100,11 @@ message GetTopicConfigurationRequest {
     string topic = 2;
 }
 message GetTopicConfigurationResponse {
-    int32 partitions = 3;
+    TopicConfiguration configuration = 1;
+}
+
+message TopicConfiguration {
+    int32 partition_count = 1;
+    string collection = 2;
+    string replication = 3;
 }
diff --git a/weed/pb/messaging_pb/messaging.pb.go b/weed/pb/messaging_pb/messaging.pb.go
index eb20e69d6..181f042df 100644
--- a/weed/pb/messaging_pb/messaging.pb.go
+++ b/weed/pb/messaging_pb/messaging.pb.go
@@ -18,6 +18,7 @@ It has these top-level messages:
 	ConfigureTopicResponse
 	GetTopicConfigurationRequest
 	GetTopicConfigurationResponse
+	TopicConfiguration
 */
 package messaging_pb
 
@@ -396,10 +397,9 @@ func (m *PublishResponse_RedirectMessage) GetNewBroker() string {
 }
 
 type ConfigureTopicRequest struct {
-	Namespace      string `protobuf:"bytes,1,opt,name=namespace" json:"namespace,omitempty"`
-	Topic          string `protobuf:"bytes,2,opt,name=topic" json:"topic,omitempty"`
-	PartitionCount int32  `protobuf:"varint,3,opt,name=partition_count,json=partitionCount" json:"partition_count,omitempty"`
-	Collection     string `protobuf:"bytes,4,opt,name=collection" json:"collection,omitempty"`
+	Namespace     string              `protobuf:"bytes,1,opt,name=namespace" json:"namespace,omitempty"`
+	Topic         string              `protobuf:"bytes,2,opt,name=topic" json:"topic,omitempty"`
+	Configuration *TopicConfiguration `protobuf:"bytes,3,opt,name=configuration" json:"configuration,omitempty"`
 }
 
 func (m *ConfigureTopicRequest) Reset()                    { *m = ConfigureTopicRequest{} }
@@ -421,18 +421,11 @@ func (m *ConfigureTopicRequest) GetTopic() string {
 	return ""
 }
 
-func (m *ConfigureTopicRequest) GetPartitionCount() int32 {
+func (m *ConfigureTopicRequest) GetConfiguration() *TopicConfiguration {
 	if m != nil {
-		return m.PartitionCount
-	}
-	return 0
-}
-
-func (m *ConfigureTopicRequest) GetCollection() string {
-	if m != nil {
-		return m.Collection
+		return m.Configuration
 	}
-	return ""
+	return nil
 }
 
 type ConfigureTopicResponse struct {
@@ -468,7 +461,7 @@ func (m *GetTopicConfigurationRequest) GetTopic() string {
 }
 
 type GetTopicConfigurationResponse struct {
-	Partitions int32 `protobuf:"varint,3,opt,name=partitions" json:"partitions,omitempty"`
+	Configuration *TopicConfiguration `protobuf:"bytes,1,opt,name=configuration" json:"configuration,omitempty"`
 }
 
 func (m *GetTopicConfigurationResponse) Reset()                    { *m = GetTopicConfigurationResponse{} }
@@ -476,13 +469,45 @@ func (m *GetTopicConfigurationResponse) String() string            { return prot
 func (*GetTopicConfigurationResponse) ProtoMessage()               {}
 func (*GetTopicConfigurationResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{8} }
 
-func (m *GetTopicConfigurationResponse) GetPartitions() int32 {
+func (m *GetTopicConfigurationResponse) GetConfiguration() *TopicConfiguration {
 	if m != nil {
-		return m.Partitions
+		return m.Configuration
+	}
+	return nil
+}
+
+type TopicConfiguration struct {
+	PartitionCount int32  `protobuf:"varint,1,opt,name=partition_count,json=partitionCount" json:"partition_count,omitempty"`
+	Collection     string `protobuf:"bytes,2,opt,name=collection" json:"collection,omitempty"`
+	Replication    string `protobuf:"bytes,3,opt,name=replication" json:"replication,omitempty"`
+}
+
+func (m *TopicConfiguration) Reset()                    { *m = TopicConfiguration{} }
+func (m *TopicConfiguration) String() string            { return proto.CompactTextString(m) }
+func (*TopicConfiguration) ProtoMessage()               {}
+func (*TopicConfiguration) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{9} }
+
+func (m *TopicConfiguration) GetPartitionCount() int32 {
+	if m != nil {
+		return m.PartitionCount
 	}
 	return 0
 }
 
+func (m *TopicConfiguration) GetCollection() string {
+	if m != nil {
+		return m.Collection
+	}
+	return ""
+}
+
+func (m *TopicConfiguration) GetReplication() string {
+	if m != nil {
+		return m.Replication
+	}
+	return ""
+}
+
 func init() {
 	proto.RegisterType((*SubscriberMessage)(nil), "messaging_pb.SubscriberMessage")
 	proto.RegisterType((*SubscriberMessage_InitMessage)(nil), "messaging_pb.SubscriberMessage.InitMessage")
@@ -500,6 +525,7 @@ func init() {
 	proto.RegisterType((*ConfigureTopicResponse)(nil), "messaging_pb.ConfigureTopicResponse")
 	proto.RegisterType((*GetTopicConfigurationRequest)(nil), "messaging_pb.GetTopicConfigurationRequest")
 	proto.RegisterType((*GetTopicConfigurationResponse)(nil), "messaging_pb.GetTopicConfigurationResponse")
+	proto.RegisterType((*TopicConfiguration)(nil), "messaging_pb.TopicConfiguration")
 	proto.RegisterEnum("messaging_pb.SubscriberMessage_InitMessage_StartPosition", SubscriberMessage_InitMessage_StartPosition_name, SubscriberMessage_InitMessage_StartPosition_value)
 }
 
@@ -742,56 +768,57 @@ var _SeaweedMessaging_serviceDesc = grpc.ServiceDesc{
 func init() { proto.RegisterFile("messaging.proto", fileDescriptor0) }
 
 var fileDescriptor0 = []byte{
-	// 802 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xb4, 0x56, 0x5d, 0x4f, 0xdb, 0x48,
-	0x14, 0xc5, 0x76, 0x12, 0xf0, 0xcd, 0xe7, 0x8e, 0x96, 0x55, 0xe4, 0x05, 0x36, 0x32, 0x2b, 0x6d,
-	0xb6, 0xa8, 0x16, 0x4a, 0x55, 0x89, 0x22, 0x24, 0x94, 0xd0, 0x88, 0x46, 0x22, 0x25, 0x9a, 0xe4,
-	0xb5, 0x8a, 0x26, 0xce, 0x00, 0x56, 0x12, 0x3b, 0xf5, 0x4c, 0x8a, 0x78, 0x6e, 0x5f, 0xfb, 0xd6,
-	0x5f, 0x52, 0xa9, 0x3f, 0xa0, 0x7d, 0xef, 0x7f, 0xaa, 0x3c, 0xfe, 0x88, 0x1d, 0x42, 0xa0, 0x48,
-	0xbc, 0xd9, 0xd7, 0xe7, 0xdc, 0x73, 0xe7, 0xde, 0x33, 0xe3, 0x81, 0xe2, 0x84, 0x32, 0x46, 0x2e,
-	0x2d, 0xfb, 0xd2, 0x98, 0xba, 0x0e, 0x77, 0x50, 0x2e, 0x0a, 0xf4, 0xa7, 0x03, 0xfd, 0x63, 0x0a,
-	0xfe, 0xe8, 0xce, 0x06, 0xcc, 0x74, 0xad, 0x01, 0x75, 0xdb, 0xe2, 0x13, 0x45, 0xc7, 0x90, 0xb2,
-	0x6c, 0x8b, 0x97, 0xa5, 0x8a, 0x54, 0xcd, 0xd6, 0xf6, 0x8c, 0x38, 0xc5, 0xb8, 0x05, 0x37, 0x5a,
-	0xb6, 0xc5, 0x83, 0x67, 0x2c, 0x88, 0xe8, 0x08, 0x14, 0x62, 0x8e, 0xca, 0xb2, 0xe0, 0x3f, 0xbb,
-	0x8f, 0x5f, 0x37, 0x47, 0x21, 0xdd, 0xa3, 0x69, 0xdf, 0x65, 0xc8, 0xc6, 0x72, 0xa2, 0x2d, 0x50,
-	0x6d, 0x32, 0xa1, 0x6c, 0x4a, 0x4c, 0x2a, 0x6a, 0x52, 0xf1, 0x3c, 0x80, 0xfe, 0x84, 0x34, 0x77,
-	0xa6, 0x96, 0x29, 0xd4, 0x54, 0xec, 0xbf, 0x78, 0x9c, 0x29, 0x71, 0xb9, 0xc5, 0x2d, 0xc7, 0x2e,
-	0x2b, 0x15, 0xa9, 0x9a, 0xc6, 0xf3, 0x00, 0xea, 0x43, 0x9e, 0x71, 0xe2, 0xf2, 0x8e, 0xc3, 0x7c,
-	0x44, 0xaa, 0x22, 0x55, 0x0b, 0xb5, 0x57, 0xbf, 0xb1, 0x52, 0xa3, 0x1b, 0x4f, 0x80, 0x93, 0xf9,
-	0x50, 0x05, 0xb2, 0xdc, 0x9a, 0x50, 0xc6, 0xc9, 0x64, 0xfa, 0x96, 0x95, 0xd3, 0x15, 0xa9, 0xaa,
-	0xe0, 0x78, 0x08, 0xed, 0x42, 0x9e, 0x45, 0xf9, 0xfb, 0xd6, 0xb0, 0x9c, 0x11, 0xe5, 0xe7, 0xe6,
-	0xc1, 0xd6, 0x50, 0x3f, 0x80, 0x7c, 0x42, 0x06, 0x01, 0x64, 0xce, 0xea, 0xbd, 0x66, 0xb7, 0x57,
-	0x5a, 0x43, 0x39, 0xd8, 0x68, 0xd6, 0xf1, 0x59, 0xcb, 0x7b, 0x93, 0x50, 0x1e, 0xd4, 0x5e, 0xab,
-	0xdd, 0xec, 0xf6, 0xea, 0xed, 0x4e, 0x49, 0xd6, 0xf6, 0x00, 0xe6, 0x6d, 0x45, 0xdb, 0x00, 0xfe,
-	0xca, 0xa8, 0xa7, 0x24, 0x89, 0x6a, 0xd4, 0x20, 0xd2, 0x1a, 0xea, 0x3f, 0x25, 0x58, 0x0f, 0xa1,
-	0x15, 0x50, 0xa3, 0x32, 0x7d, 0x64, 0x43, 0xde, 0x97, 0xf0, 0x3c, 0x88, 0x4a, 0xa0, 0x8c, 0xe8,
-	0x8d, 0x68, 0x77, 0x0e, 0x7b, 0x8f, 0xde, 0x08, 0x3e, 0x90, 0xf1, 0x8c, 0x8a, 0x46, 0xe7, 0xb0,
-	0xff, 0x82, 0x8e, 0x60, 0xfd, 0x8a, 0x92, 0x21, 0x75, 0x59, 0x39, 0x55, 0x51, 0xaa, 0xd9, 0x9a,
-	0x9e, 0x6c, 0x6f, 0xd8, 0xc8, 0x37, 0x3e, 0xa8, 0x69, 0x73, 0xf7, 0x06, 0x87, 0x14, 0xed, 0x10,
-	0x72, 0xf1, 0x0f, 0xa1, 0xaa, 0x3f, 0xfe, 0xa4, 0xaa, 0x1c, 0x53, 0x3d, 0x94, 0x0f, 0x24, 0xfd,
-	0x9b, 0x04, 0xf9, 0x86, 0xeb, 0x8c, 0xe6, 0x8e, 0xfe, 0x1f, 0x52, 0x43, 0xc2, 0x49, 0xe0, 0xe8,
-	0xcd, 0xa5, 0x85, 0x60, 0x01, 0x41, 0xa7, 0xb0, 0xe1, 0xd2, 0xa1, 0xe5, 0x52, 0x93, 0x07, 0x06,
-	0x5e, 0xd8, 0x00, 0x89, 0xcc, 0x06, 0x0e, 0xb0, 0x61, 0x92, 0x88, 0xac, 0xed, 0x43, 0x71, 0xe1,
-	0xa3, 0x37, 0x07, 0x9b, 0x5e, 0xf7, 0x07, 0x22, 0x43, 0x64, 0x65, 0x7a, 0xed, 0xa7, 0xd4, 0xbf,
-	0x2a, 0x50, 0xe8, 0xcc, 0x06, 0x63, 0x8b, 0x5d, 0x61, 0xfa, 0x7e, 0x46, 0x99, 0xb7, 0x93, 0xe2,
-	0x5b, 0xb1, 0x9a, 0xac, 0x24, 0x89, 0x5d, 0xba, 0x0f, 0xfd, 0x65, 0xcb, 0x0f, 0x60, 0xbf, 0x26,
-	0x9c, 0x24, 0x3a, 0xa1, 0xf5, 0x9f, 0x78, 0x1b, 0x6a, 0x3f, 0x24, 0xc8, 0xc6, 0x64, 0xe3, 0x33,
-	0xce, 0xad, 0x98, 0x31, 0x3a, 0x9f, 0x3b, 0x4b, 0x11, 0xce, 0x7a, 0xf9, 0xd0, 0x95, 0x3d, 0x81,
-	0xd9, 0x3e, 0xcb, 0x50, 0x8c, 0x04, 0xd9, 0xd4, 0xb1, 0x19, 0x45, 0x27, 0x90, 0x31, 0x1d, 0xfb,
-	0xc2, 0xba, 0x5c, 0x7e, 0x84, 0x2e, 0xc0, 0x8d, 0x13, 0x81, 0x0d, 0x9b, 0x1f, 0x50, 0x51, 0xeb,
-	0x96, 0x11, 0x9f, 0xaf, 0x4e, 0x73, 0xb7, 0x15, 0x0f, 0x20, 0x9f, 0xd0, 0x40, 0xff, 0x41, 0x31,
-	0x1a, 0x43, 0xdf, 0x74, 0x66, 0xb6, 0xef, 0xb0, 0x34, 0x2e, 0x44, 0xe1, 0x13, 0x2f, 0xfa, 0x08,
-	0x13, 0x7f, 0x91, 0x60, 0xd3, 0x17, 0x9b, 0xb9, 0xb4, 0xe7, 0xb9, 0x20, 0xf4, 0xf2, 0x63, 0x0c,
-	0xb4, 0xa4, 0x50, 0x65, 0x59, 0xa1, 0x68, 0x07, 0xc0, 0x74, 0xc6, 0x63, 0x6a, 0x46, 0xe7, 0xb9,
-	0x8a, 0x63, 0x11, 0xbd, 0x0c, 0x7f, 0x2d, 0x56, 0xe5, 0xb7, 0x4d, 0xc7, 0xb0, 0x75, 0x4a, 0xb9,
-	0x88, 0x85, 0x08, 0x22, 0xce, 0xf4, 0xc7, 0x97, 0xad, 0x1f, 0xc3, 0xf6, 0x1d, 0x39, 0x03, 0x87,
-	0xec, 0x00, 0x44, 0x0b, 0x60, 0xc1, 0x92, 0x62, 0x91, 0xda, 0x27, 0x05, 0x4a, 0x5d, 0x4a, 0xae,
-	0x29, 0x1d, 0xb6, 0xc3, 0x99, 0xa3, 0x73, 0x50, 0xa3, 0x7f, 0x12, 0xfa, 0xe7, 0x9e, 0x9f, 0x95,
-	0xf6, 0xf7, 0x8a, 0x63, 0x4b, 0x5f, 0xab, 0x4a, 0xfb, 0x12, 0x3a, 0x83, 0xf5, 0xc0, 0x44, 0x68,
-	0x6b, 0xd5, 0x16, 0xd2, 0xb6, 0x57, 0x3a, 0x2f, 0xc8, 0xf6, 0x0e, 0x0a, 0xc9, 0x16, 0xa3, 0xdd,
-	0x24, 0x6d, 0xa9, 0x2d, 0xb4, 0x7f, 0x57, 0x83, 0x42, 0x09, 0xe4, 0xc2, 0xe6, 0xd2, 0x9e, 0xa2,
-	0x85, 0x0b, 0xc6, 0xaa, 0x61, 0x6a, 0x7b, 0x0f, 0xc2, 0x86, 0x9a, 0x0d, 0x1d, 0x4a, 0xcc, 0x9f,
-	0xc2, 0x05, 0x33, 0xcc, 0xb1, 0x45, 0x6d, 0xde, 0x28, 0x44, 0x03, 0xe9, 0x78, 0x37, 0xaa, 0x41,
-	0x46, 0x5c, 0xac, 0x5e, 0xfc, 0x0a, 0x00, 0x00, 0xff, 0xff, 0x5f, 0x31, 0x28, 0xa2, 0x6b, 0x09,
-	0x00, 0x00,
+	// 829 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xb4, 0x56, 0x4d, 0x8f, 0xe3, 0x44,
+	0x10, 0xdd, 0xb6, 0x93, 0xcc, 0xba, 0xf2, 0x49, 0x8b, 0x41, 0x91, 0x99, 0x01, 0xcb, 0x8b, 0x44,
+	0x60, 0x84, 0x35, 0x0a, 0x42, 0x1a, 0x56, 0x2b, 0xa1, 0x24, 0x84, 0x25, 0xd2, 0x84, 0x8d, 0x3a,
+	0xb9, 0xa2, 0xc8, 0x71, 0x7a, 0xb3, 0x56, 0x12, 0xdb, 0xb8, 0x3b, 0x8c, 0xe6, 0xc4, 0x01, 0xae,
+	0x9c, 0xf8, 0x27, 0x48, 0xfc, 0x00, 0xb8, 0xf3, 0x9f, 0x90, 0xdb, 0xdf, 0x49, 0x26, 0x13, 0x22,
+	0xcd, 0xcd, 0x2e, 0xbf, 0x7a, 0xaf, 0xba, 0xea, 0x95, 0x6d, 0xa8, 0xaf, 0x29, 0x63, 0xe6, 0xc2,
+	0x76, 0x16, 0x86, 0xe7, 0xbb, 0xdc, 0xc5, 0x95, 0x24, 0x30, 0xf5, 0x66, 0xfa, 0xaf, 0x05, 0x78,
+	0x6f, 0xbc, 0x99, 0x31, 0xcb, 0xb7, 0x67, 0xd4, 0x1f, 0x8a, 0x47, 0x14, 0x7f, 0x03, 0x05, 0xdb,
+	0xb1, 0x79, 0x13, 0x69, 0xa8, 0x55, 0x6e, 0x5f, 0x19, 0xd9, 0x14, 0x63, 0x07, 0x6e, 0x0c, 0x1c,
+	0x9b, 0x47, 0xd7, 0x44, 0x24, 0xe2, 0x57, 0x20, 0x9b, 0xd6, 0xb2, 0x29, 0x89, 0xfc, 0xcf, 0x1f,
+	0xcb, 0xef, 0x58, 0xcb, 0x38, 0x3d, 0x48, 0x53, 0xff, 0x96, 0xa0, 0x9c, 0xe1, 0xc4, 0x17, 0xa0,
+	0x38, 0xe6, 0x9a, 0x32, 0xcf, 0xb4, 0xa8, 0xa8, 0x49, 0x21, 0x69, 0x00, 0xbf, 0x0f, 0x45, 0xee,
+	0x7a, 0xb6, 0x25, 0xd4, 0x14, 0x12, 0xde, 0x04, 0x39, 0x9e, 0xe9, 0x73, 0x9b, 0xdb, 0xae, 0xd3,
+	0x94, 0x35, 0xd4, 0x2a, 0x92, 0x34, 0x80, 0xa7, 0x50, 0x65, 0xdc, 0xf4, 0xf9, 0xc8, 0x65, 0x21,
+	0xa2, 0xa0, 0xa1, 0x56, 0xad, 0xfd, 0xf5, 0xff, 0x38, 0xa9, 0x31, 0xce, 0x12, 0x90, 0x3c, 0x1f,
+	0xd6, 0xa0, 0xcc, 0xed, 0x35, 0x65, 0xdc, 0x5c, 0x7b, 0x3f, 0xb0, 0x66, 0x51, 0x43, 0x2d, 0x99,
+	0x64, 0x43, 0xf8, 0x05, 0x54, 0x59, 0xc2, 0x3f, 0xb5, 0xe7, 0xcd, 0x92, 0x28, 0xbf, 0x92, 0x06,
+	0x07, 0x73, 0xfd, 0x06, 0xaa, 0x39, 0x19, 0x0c, 0x50, 0xba, 0xed, 0x4c, 0xfa, 0xe3, 0x49, 0xe3,
+	0x19, 0xae, 0xc0, 0xf3, 0x7e, 0x87, 0xdc, 0x0e, 0x82, 0x3b, 0x84, 0xab, 0xa0, 0x4c, 0x06, 0xc3,
+	0xfe, 0x78, 0xd2, 0x19, 0x8e, 0x1a, 0x92, 0x7a, 0x05, 0x90, 0xb6, 0x15, 0x5f, 0x02, 0x84, 0x27,
+	0xa3, 0x81, 0x12, 0x12, 0xd5, 0x28, 0x51, 0x64, 0x30, 0xd7, 0xff, 0x45, 0x70, 0x16, 0x43, 0x35,
+	0x50, 0x92, 0x32, 0x43, 0x64, 0x57, 0xba, 0x46, 0x24, 0x0d, 0xe2, 0x06, 0xc8, 0x4b, 0x7a, 0x2f,
+	0xda, 0x5d, 0x21, 0xc1, 0x65, 0x30, 0x82, 0x9f, 0xcd, 0xd5, 0x86, 0x8a, 0x46, 0x57, 0x48, 0x78,
+	0x83, 0x5f, 0xc1, 0xd9, 0x3b, 0x6a, 0xce, 0xa9, 0xcf, 0x9a, 0x05, 0x4d, 0x6e, 0x95, 0xdb, 0x7a,
+	0xbe, 0xbd, 0x71, 0x23, 0xbf, 0x0f, 0x41, 0x7d, 0x87, 0xfb, 0xf7, 0x24, 0x4e, 0x51, 0x5f, 0x42,
+	0x25, 0xfb, 0x20, 0x56, 0x0d, 0xc7, 0x9f, 0x57, 0x95, 0x32, 0xaa, 0x2f, 0xa5, 0x1b, 0xa4, 0xff,
+	0x85, 0xa0, 0xda, 0xf5, 0xdd, 0x65, 0xea, 0xe8, 0xcf, 0xa0, 0x30, 0x37, 0xb9, 0x19, 0x39, 0xfa,
+	0x7c, 0x6f, 0x21, 0x44, 0x40, 0xf0, 0x6b, 0x78, 0xee, 0xd3, 0xb9, 0xed, 0x53, 0x8b, 0x47, 0x06,
+	0xde, 0x5a, 0x80, 0x1c, 0xb3, 0x41, 0x22, 0x6c, 0x4c, 0x92, 0x24, 0xab, 0xd7, 0x50, 0xdf, 0x7a,
+	0x18, 0xcc, 0xc1, 0xa1, 0x77, 0xd3, 0x99, 0x60, 0x48, 0xac, 0x4c, 0xef, 0x42, 0x4a, 0xfd, 0x4f,
+	0x19, 0x6a, 0xa3, 0xcd, 0x6c, 0x65, 0xb3, 0x77, 0x84, 0xfe, 0xb4, 0xa1, 0x2c, 0xd8, 0xa4, 0xec,
+	0x2a, 0xb6, 0xf2, 0x95, 0xe4, 0xb1, 0x7b, 0xf7, 0x30, 0x3c, 0xb6, 0x74, 0x44, 0xf6, 0xb7, 0x26,
+	0x37, 0x73, 0x9d, 0x50, 0xa7, 0x4f, 0xbc, 0x86, 0xea, 0x3f, 0x08, 0xca, 0x19, 0xd9, 0xec, 0x8c,
+	0x2b, 0x07, 0x66, 0x8c, 0xdf, 0xa4, 0xce, 0x92, 0x85, 0xb3, 0xbe, 0x3a, 0xf6, 0x64, 0x4f, 0x60,
+	0xb6, 0xdf, 0x25, 0xa8, 0x27, 0x82, 0xcc, 0x73, 0x1d, 0x46, 0x71, 0x0f, 0x4a, 0x96, 0xeb, 0xbc,
+	0xb5, 0x17, 0xfb, 0x5f, 0xa1, 0x5b, 0x70, 0xa3, 0x27, 0xb0, 0x71, 0xf3, 0xa3, 0x54, 0x3c, 0xd8,
+	0x31, 0xe2, 0x17, 0x87, 0x69, 0x1e, 0xb6, 0xe2, 0x0d, 0x54, 0x73, 0x1a, 0xf8, 0x53, 0xa8, 0x27,
+	0x63, 0x98, 0x5a, 0xee, 0xc6, 0x09, 0x1d, 0x56, 0x24, 0xb5, 0x24, 0xdc, 0x0b, 0xa2, 0x27, 0x98,
+	0xf8, 0x0f, 0x04, 0xe7, 0xa1, 0xd8, 0xc6, 0xa7, 0x93, 0xc0, 0x05, 0xb1, 0x97, 0x4f, 0x31, 0xd0,
+	0x77, 0x50, 0xb5, 0x22, 0x32, 0x33, 0x31, 0x51, 0xb9, 0xad, 0xe5, 0x3b, 0x21, 0x64, 0x7a, 0x59,
+	0x1c, 0xc9, 0xa7, 0xe9, 0x4d, 0xf8, 0x60, 0xbb, 0xa8, 0xb0, 0x6b, 0x3a, 0x81, 0x8b, 0xd7, 0x94,
+	0xef, 0x61, 0x38, 0xbd, 0x6a, 0x7d, 0x01, 0x97, 0x0f, 0x70, 0x46, 0x06, 0xd9, 0x39, 0x16, 0x3a,
+	0xed, 0x58, 0xbf, 0x00, 0xde, 0x05, 0x1d, 0x3d, 0x5d, 0xfc, 0x11, 0x80, 0xe5, 0xae, 0x56, 0xd4,
+	0x12, 0x35, 0x84, 0x47, 0xc8, 0x44, 0x82, 0xcf, 0x98, 0x4f, 0xbd, 0x95, 0x6d, 0xa5, 0xbd, 0x57,
+	0x48, 0x36, 0xd4, 0xfe, 0x4d, 0x86, 0xc6, 0x98, 0x9a, 0x77, 0x94, 0xce, 0x87, 0x71, 0xe9, 0xf8,
+	0x0d, 0x28, 0xc9, 0xb7, 0x13, 0x7f, 0xfc, 0xc8, 0x47, 0x55, 0xfd, 0xf0, 0xc0, 0xeb, 0x55, 0x7f,
+	0xd6, 0x42, 0xd7, 0x08, 0xdf, 0xc2, 0x59, 0x64, 0x76, 0x7c, 0x71, 0x68, 0xd5, 0xd5, 0xcb, 0x83,
+	0x1b, 0x12, 0xb1, 0xfd, 0x08, 0xb5, 0xbc, 0x17, 0xf0, 0x8b, 0x7c, 0xda, 0x5e, 0xfb, 0xaa, 0x9f,
+	0x1c, 0x06, 0xc5, 0x12, 0xd8, 0x87, 0xf3, 0xbd, 0xc3, 0xc7, 0x5b, 0x3f, 0x42, 0x87, 0x5c, 0xa7,
+	0x5e, 0x1d, 0x85, 0x8d, 0x35, 0xbb, 0x3a, 0x34, 0x58, 0x38, 0x85, 0xb7, 0xcc, 0xb0, 0x56, 0x36,
+	0x75, 0x78, 0xb7, 0x96, 0x0c, 0x64, 0x14, 0xfc, 0xf9, 0xcd, 0x4a, 0xe2, 0x07, 0xf0, 0xcb, 0xff,
+	0x02, 0x00, 0x00, 0xff, 0xff, 0xed, 0x8d, 0x77, 0xac, 0x13, 0x0a, 0x00, 0x00,
 }

From ffa69b24ae0b5daf6cb040c95bb821c25ba15461 Mon Sep 17 00:00:00 2001
From: Chris Lu <chris.lu@gmail.com>
Date: Fri, 17 Apr 2020 10:01:11 -0700
Subject: [PATCH 76/81] fix travis

---
 weed/messaging/broker_append.go | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/weed/messaging/broker_append.go b/weed/messaging/broker_append.go
index 07b3755c0..a0c0e8614 100644
--- a/weed/messaging/broker_append.go
+++ b/weed/messaging/broker_append.go
@@ -58,7 +58,6 @@ func (broker *MessageBroker) appendToFile(targetFile string, topicConfig *messag
 func (broker *MessageBroker) assignAndUpload(topicConfig *messaging_pb.TopicConfiguration, data []byte) (*operation.AssignResult, *operation.UploadResult, error) {
 
 	var assignResult = &operation.AssignResult{}
-	var collection, replication string
 
 	// assign a volume location
 	if err := broker.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
@@ -84,8 +83,6 @@ func (broker *MessageBroker) assignAndUpload(topicConfig *messaging_pb.TopicConf
 		assignResult.PublicUrl = resp.PublicUrl
 		assignResult.Count = uint64(resp.Count)
 
-		collection, replication = resp.Collection, resp.Replication
-
 		return nil
 	}); err != nil {
 		return nil, nil, err

From 826f96b6f39839c90b906f6293e976dd86e3ea77 Mon Sep 17 00:00:00 2001
From: Chris Lu <chris.lu@gmail.com>
Date: Fri, 17 Apr 2020 10:04:28 -0700
Subject: [PATCH 77/81] fix unreachable code

---
 weed/command/filer_replication.go       | 1 -
 weed/server/filer_grpc_server_listen.go | 1 -
 weed/server/master_grpc_server.go       | 1 -
 3 files changed, 3 deletions(-)

diff --git a/weed/command/filer_replication.go b/weed/command/filer_replication.go
index 737f0d24a..40f2b570b 100644
--- a/weed/command/filer_replication.go
+++ b/weed/command/filer_replication.go
@@ -121,7 +121,6 @@ func runFilerReplicate(cmd *Command, args []string) bool {
 		}
 	}
 
-	return true
 }
 
 func validateOneEnabledInput(config *viper.Viper) {
diff --git a/weed/server/filer_grpc_server_listen.go b/weed/server/filer_grpc_server_listen.go
index 76bf834ea..ea3abe28f 100644
--- a/weed/server/filer_grpc_server_listen.go
+++ b/weed/server/filer_grpc_server_listen.go
@@ -65,7 +65,6 @@ func (fs *FilerServer) SubscribeMetadata(req *filer_pb.SubscribeMetadataRequest,
 		fs.listenersLock.Unlock()
 	}
 
-	return nil
 }
 
 func (fs *FilerServer) addClient(clientType string, clientAddress string) (clientName string) {
diff --git a/weed/server/master_grpc_server.go b/weed/server/master_grpc_server.go
index a8da9ab2e..035ed4435 100644
--- a/weed/server/master_grpc_server.go
+++ b/weed/server/master_grpc_server.go
@@ -243,7 +243,6 @@ func (ms *MasterServer) KeepConnected(stream master_pb.Seaweed_KeepConnectedServ
 		}
 	}
 
-	return nil
 }
 
 func (ms *MasterServer) informNewLeader(stream master_pb.Seaweed_KeepConnectedServer) error {

From e0f599656008b2075dc874271597027129689e4d Mon Sep 17 00:00:00 2001
From: Chris Lu <chris.lu@gmail.com>
Date: Fri, 17 Apr 2020 10:04:41 -0700
Subject: [PATCH 78/81] fix "call of Unmarshal passes non-pointer as second
 argument"

---
 weed/util/http_util.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/weed/util/http_util.go b/weed/util/http_util.go
index 4b1a7b895..5df79a7be 100644
--- a/weed/util/http_util.go
+++ b/weed/util/http_util.go
@@ -117,7 +117,7 @@ func Delete(url string, jwt string) error {
 		return nil
 	}
 	m := make(map[string]interface{})
-	if e := json.Unmarshal(body, m); e == nil {
+	if e := json.Unmarshal(body, &m); e == nil {
 		if s, ok := m["error"].(string); ok {
 			return errors.New(s)
 		}

From 2e2537a9ea442476efb12e4caa2663b804cf4ee1 Mon Sep 17 00:00:00 2001
From: Chris Lu <chris.lu@gmail.com>
Date: Fri, 17 Apr 2020 22:35:44 -0700
Subject: [PATCH 79/81] rename files

---
 .../{msg_broker_grpc_server.go => broker_grpc_server.go}          | 0
 weed/messaging/{msg_broker_server.go => broker_server.go}         | 0
 2 files changed, 0 insertions(+), 0 deletions(-)
 rename weed/messaging/{msg_broker_grpc_server.go => broker_grpc_server.go} (100%)
 rename weed/messaging/{msg_broker_server.go => broker_server.go} (100%)

diff --git a/weed/messaging/msg_broker_grpc_server.go b/weed/messaging/broker_grpc_server.go
similarity index 100%
rename from weed/messaging/msg_broker_grpc_server.go
rename to weed/messaging/broker_grpc_server.go
diff --git a/weed/messaging/msg_broker_server.go b/weed/messaging/broker_server.go
similarity index 100%
rename from weed/messaging/msg_broker_server.go
rename to weed/messaging/broker_server.go

From 5bea77010f2b290398c46a517d40fa7ad559dfed Mon Sep 17 00:00:00 2001
From: Chris Lu <chris.lu@gmail.com>
Date: Fri, 17 Apr 2020 22:39:21 -0700
Subject: [PATCH 80/81] refactor

---
 weed/messaging/broker_grpc_server.go          | 88 ------------------
 weed/messaging/broker_grpc_server_publish.go  | 90 +++++++++++++++++++
 .../messaging/broker_grpc_server_subscribe.go |  9 ++
 3 files changed, 99 insertions(+), 88 deletions(-)
 create mode 100644 weed/messaging/broker_grpc_server_publish.go
 create mode 100644 weed/messaging/broker_grpc_server_subscribe.go

diff --git a/weed/messaging/broker_grpc_server.go b/weed/messaging/broker_grpc_server.go
index f4b8321e2..0d1eb72ac 100644
--- a/weed/messaging/broker_grpc_server.go
+++ b/weed/messaging/broker_grpc_server.go
@@ -2,98 +2,10 @@ package messaging
 
 import (
 	"context"
-	"fmt"
-	"io"
-	"time"
 
-	"github.com/golang/protobuf/proto"
-
-	"github.com/chrislusf/seaweedfs/weed/filer2"
-	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/pb/messaging_pb"
-	"github.com/chrislusf/seaweedfs/weed/util/log_buffer"
 )
 
-func (broker *MessageBroker) Subscribe(server messaging_pb.SeaweedMessaging_SubscribeServer) error {
-	panic("implement me")
-}
-
-func (broker *MessageBroker) Publish(stream messaging_pb.SeaweedMessaging_PublishServer) error {
-
-	// process initial request
-	in, err := stream.Recv()
-	if err == io.EOF {
-		return nil
-	}
-	if err != nil {
-		return err
-	}
-	namespace, topic, partition := in.Init.Namespace, in.Init.Topic, in.Init.Partition
-
-	updatesChan := make(chan int32)
-
-	// TODO look it up
-	topicConfig := &messaging_pb.TopicConfiguration{
-
-	}
-
-	go func() {
-		for update := range updatesChan {
-			if err := stream.Send(&messaging_pb.PublishResponse{
-				Config: &messaging_pb.PublishResponse_ConfigMessage{
-					PartitionCount: update,
-				},
-			}); err != nil {
-				glog.V(0).Infof("err sending publish response: %v", err)
-				return
-			}
-		}
-	}()
-
-	logBuffer := log_buffer.NewLogBuffer(time.Minute, func(startTime, stopTime time.Time, buf []byte) {
-
-		targetFile := fmt.Sprintf(
-			"%s/%s/%s/%04d-%02d-%02d/%02d-%02d.part%02d",
-			filer2.TopicsDir, namespace, topic,
-			startTime.Year(), startTime.Month(), startTime.Day(), startTime.Hour(), startTime.Minute(),
-			partition,
-		)
-
-		if err := broker.appendToFile(targetFile, topicConfig, buf); err != nil {
-			glog.V(0).Infof("log write failed %s: %v", targetFile, err)
-		}
-
-	}, func() {
-		// notify subscribers
-	})
-
-	for {
-		in, err := stream.Recv()
-		if err == io.EOF {
-			return nil
-		}
-		if err != nil {
-			return err
-		}
-
-		m := &messaging_pb.Message{
-			Timestamp: time.Now().UnixNano(),
-			Key:       in.Data.Key,
-			Value:     in.Data.Value,
-			Headers:   in.Data.Headers,
-		}
-
-		data, err := proto.Marshal(m)
-		if err != nil {
-			glog.Errorf("marshall error: %v\n", err)
-			continue
-		}
-
-		logBuffer.AddToBuffer(in.Data.Key, data)
-
-	}
-}
-
 func (broker *MessageBroker) ConfigureTopic(c context.Context, request *messaging_pb.ConfigureTopicRequest) (*messaging_pb.ConfigureTopicResponse, error) {
 	panic("implement me")
 }
diff --git a/weed/messaging/broker_grpc_server_publish.go b/weed/messaging/broker_grpc_server_publish.go
new file mode 100644
index 000000000..db3bf0764
--- /dev/null
+++ b/weed/messaging/broker_grpc_server_publish.go
@@ -0,0 +1,90 @@
+package messaging
+
+import (
+	"fmt"
+	"io"
+	"time"
+
+	"github.com/golang/protobuf/proto"
+
+	"github.com/chrislusf/seaweedfs/weed/filer2"
+	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/pb/messaging_pb"
+	"github.com/chrislusf/seaweedfs/weed/util/log_buffer"
+)
+
+func (broker *MessageBroker) Publish(stream messaging_pb.SeaweedMessaging_PublishServer) error {
+
+	// process initial request
+	in, err := stream.Recv()
+	if err == io.EOF {
+		return nil
+	}
+	if err != nil {
+		return err
+	}
+	namespace, topic, partition := in.Init.Namespace, in.Init.Topic, in.Init.Partition
+
+	updatesChan := make(chan int32)
+
+	// TODO look it up
+	topicConfig := &messaging_pb.TopicConfiguration{
+
+	}
+
+	go func() {
+		for update := range updatesChan {
+			if err := stream.Send(&messaging_pb.PublishResponse{
+				Config: &messaging_pb.PublishResponse_ConfigMessage{
+					PartitionCount: update,
+				},
+			}); err != nil {
+				glog.V(0).Infof("err sending publish response: %v", err)
+				return
+			}
+		}
+	}()
+
+	logBuffer := log_buffer.NewLogBuffer(time.Minute, func(startTime, stopTime time.Time, buf []byte) {
+
+		targetFile := fmt.Sprintf(
+			"%s/%s/%s/%04d-%02d-%02d/%02d-%02d.part%02d",
+			filer2.TopicsDir, namespace, topic,
+			startTime.Year(), startTime.Month(), startTime.Day(), startTime.Hour(), startTime.Minute(),
+			partition,
+		)
+
+		if err := broker.appendToFile(targetFile, topicConfig, buf); err != nil {
+			glog.V(0).Infof("log write failed %s: %v", targetFile, err)
+		}
+
+	}, func() {
+		// notify subscribers
+	})
+
+	for {
+		in, err := stream.Recv()
+		if err == io.EOF {
+			return nil
+		}
+		if err != nil {
+			return err
+		}
+
+		m := &messaging_pb.Message{
+			Timestamp: time.Now().UnixNano(),
+			Key:       in.Data.Key,
+			Value:     in.Data.Value,
+			Headers:   in.Data.Headers,
+		}
+
+		data, err := proto.Marshal(m)
+		if err != nil {
+			glog.Errorf("marshall error: %v\n", err)
+			continue
+		}
+
+		logBuffer.AddToBuffer(in.Data.Key, data)
+
+	}
+}
diff --git a/weed/messaging/broker_grpc_server_subscribe.go b/weed/messaging/broker_grpc_server_subscribe.go
new file mode 100644
index 000000000..137fcac8a
--- /dev/null
+++ b/weed/messaging/broker_grpc_server_subscribe.go
@@ -0,0 +1,9 @@
+package messaging
+
+import (
+	"github.com/chrislusf/seaweedfs/weed/pb/messaging_pb"
+)
+
+func (broker *MessageBroker) Subscribe(server messaging_pb.SeaweedMessaging_SubscribeServer) error {
+	panic("implement me")
+}

From 11f5a6d91346e5f3cbf3b46e0a660e231c5c2998 Mon Sep 17 00:00:00 2001
From: Chris Lu <chris.lu@gmail.com>
Date: Sat, 18 Apr 2020 01:12:01 -0700
Subject: [PATCH 81/81] messaging can compile now

---
 weed/command/msg_broker.go                    |  4 +-
 weed/messaging/{ => broker}/broker_append.go  |  2 +-
 .../{ => broker}/broker_grpc_server.go        |  2 +-
 .../broker_grpc_server_publish.go             | 51 ++++++-----
 .../broker/broker_grpc_server_subscribe.go    | 88 +++++++++++++++++++
 weed/messaging/{ => broker}/broker_server.go  |  4 +-
 weed/messaging/broker/topic_lock.go           | 80 +++++++++++++++++
 .../messaging/broker_grpc_server_subscribe.go |  9 --
 8 files changed, 205 insertions(+), 35 deletions(-)
 rename weed/messaging/{ => broker}/broker_append.go (99%)
 rename weed/messaging/{ => broker}/broker_grpc_server.go (96%)
 rename weed/messaging/{ => broker}/broker_grpc_server_publish.go (79%)
 create mode 100644 weed/messaging/broker/broker_grpc_server_subscribe.go
 rename weed/messaging/{ => broker}/broker_server.go (97%)
 create mode 100644 weed/messaging/broker/topic_lock.go
 delete mode 100644 weed/messaging/broker_grpc_server_subscribe.go

diff --git a/weed/command/msg_broker.go b/weed/command/msg_broker.go
index 36d164800..67ebdfb6d 100644
--- a/weed/command/msg_broker.go
+++ b/weed/command/msg_broker.go
@@ -9,7 +9,7 @@ import (
 	"google.golang.org/grpc/reflection"
 
 	"github.com/chrislusf/seaweedfs/weed/glog"
-	"github.com/chrislusf/seaweedfs/weed/messaging"
+	"github.com/chrislusf/seaweedfs/weed/messaging/broker"
 	"github.com/chrislusf/seaweedfs/weed/pb"
 	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
 	"github.com/chrislusf/seaweedfs/weed/pb/messaging_pb"
@@ -80,7 +80,7 @@ func (msgBrokerOpt *QueueOptions) startQueueServer() bool {
 		}
 	}
 
-	qs, err := messaging.NewMessageBroker(&messaging.MessageBrokerOption{
+	qs, err := broker.NewMessageBroker(&broker.MessageBrokerOption{
 		Filers:             []string{*msgBrokerOpt.filer},
 		DefaultReplication: "",
 		MaxMB:              0,
diff --git a/weed/messaging/broker_append.go b/weed/messaging/broker/broker_append.go
similarity index 99%
rename from weed/messaging/broker_append.go
rename to weed/messaging/broker/broker_append.go
index a0c0e8614..7194dfcfc 100644
--- a/weed/messaging/broker_append.go
+++ b/weed/messaging/broker/broker_append.go
@@ -1,4 +1,4 @@
-package messaging
+package broker
 
 import (
 	"context"
diff --git a/weed/messaging/broker_grpc_server.go b/weed/messaging/broker/broker_grpc_server.go
similarity index 96%
rename from weed/messaging/broker_grpc_server.go
rename to weed/messaging/broker/broker_grpc_server.go
index 0d1eb72ac..447620a6b 100644
--- a/weed/messaging/broker_grpc_server.go
+++ b/weed/messaging/broker/broker_grpc_server.go
@@ -1,4 +1,4 @@
-package messaging
+package broker
 
 import (
 	"context"
diff --git a/weed/messaging/broker_grpc_server_publish.go b/weed/messaging/broker/broker_grpc_server_publish.go
similarity index 79%
rename from weed/messaging/broker_grpc_server_publish.go
rename to weed/messaging/broker/broker_grpc_server_publish.go
index db3bf0764..20e6eb04b 100644
--- a/weed/messaging/broker_grpc_server_publish.go
+++ b/weed/messaging/broker/broker_grpc_server_publish.go
@@ -1,4 +1,4 @@
-package messaging
+package broker
 
 import (
 	"fmt"
@@ -10,7 +10,6 @@ import (
 	"github.com/chrislusf/seaweedfs/weed/filer2"
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/pb/messaging_pb"
-	"github.com/chrislusf/seaweedfs/weed/util/log_buffer"
 )
 
 func (broker *MessageBroker) Publish(stream messaging_pb.SeaweedMessaging_PublishServer) error {
@@ -23,15 +22,36 @@ func (broker *MessageBroker) Publish(stream messaging_pb.SeaweedMessaging_Publis
 	if err != nil {
 		return err
 	}
-	namespace, topic, partition := in.Init.Namespace, in.Init.Topic, in.Init.Partition
-
-	updatesChan := make(chan int32)
 
 	// TODO look it up
 	topicConfig := &messaging_pb.TopicConfiguration{
 
 	}
 
+	// get lock
+	tp := TopicPartition{
+		Namespace: in.Init.Namespace,
+		Topic:     in.Init.Topic,
+		Partition: in.Init.Partition,
+	}
+	logBuffer := broker.topicLocks.RequestPublisherLock(tp, func(startTime, stopTime time.Time, buf []byte) {
+
+		targetFile := fmt.Sprintf(
+			"%s/%s/%s/%04d-%02d-%02d/%02d-%02d.part%02d",
+			filer2.TopicsDir, tp.Namespace, tp.Topic,
+			startTime.Year(), startTime.Month(), startTime.Day(), startTime.Hour(), startTime.Minute(),
+			tp.Partition,
+		)
+
+		if err := broker.appendToFile(targetFile, topicConfig, buf); err != nil {
+			glog.V(0).Infof("log write failed %s: %v", targetFile, err)
+		}
+
+	})
+	defer broker.topicLocks.ReleaseLock(tp, true)
+
+	updatesChan := make(chan int32)
+
 	go func() {
 		for update := range updatesChan {
 			if err := stream.Send(&messaging_pb.PublishResponse{
@@ -45,23 +65,8 @@ func (broker *MessageBroker) Publish(stream messaging_pb.SeaweedMessaging_Publis
 		}
 	}()
 
-	logBuffer := log_buffer.NewLogBuffer(time.Minute, func(startTime, stopTime time.Time, buf []byte) {
-
-		targetFile := fmt.Sprintf(
-			"%s/%s/%s/%04d-%02d-%02d/%02d-%02d.part%02d",
-			filer2.TopicsDir, namespace, topic,
-			startTime.Year(), startTime.Month(), startTime.Day(), startTime.Hour(), startTime.Minute(),
-			partition,
-		)
-
-		if err := broker.appendToFile(targetFile, topicConfig, buf); err != nil {
-			glog.V(0).Infof("log write failed %s: %v", targetFile, err)
-		}
-
-	}, func() {
-		// notify subscribers
-	})
 
+	// process each message
 	for {
 		in, err := stream.Recv()
 		if err == io.EOF {
@@ -71,6 +76,10 @@ func (broker *MessageBroker) Publish(stream messaging_pb.SeaweedMessaging_Publis
 			return err
 		}
 
+		if in.Data == nil {
+			continue
+		}
+
 		m := &messaging_pb.Message{
 			Timestamp: time.Now().UnixNano(),
 			Key:       in.Data.Key,
diff --git a/weed/messaging/broker/broker_grpc_server_subscribe.go b/weed/messaging/broker/broker_grpc_server_subscribe.go
new file mode 100644
index 000000000..5a3c4f785
--- /dev/null
+++ b/weed/messaging/broker/broker_grpc_server_subscribe.go
@@ -0,0 +1,88 @@
+package broker
+
+import (
+	"io"
+	"sync"
+	"time"
+
+	"github.com/golang/protobuf/proto"
+
+	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/pb/messaging_pb"
+	"github.com/chrislusf/seaweedfs/weed/util"
+)
+
+func (broker *MessageBroker) Subscribe(stream messaging_pb.SeaweedMessaging_SubscribeServer) error {
+
+	// process initial request
+	in, err := stream.Recv()
+	if err == io.EOF {
+		return nil
+	}
+	if err != nil {
+		return err
+	}
+
+	subscriberId := in.Init.SubscriberId
+
+	// get lock
+	tp := TopicPartition{
+		Namespace: in.Init.Namespace,
+		Topic:     in.Init.Topic,
+		Partition: in.Init.Partition,
+	}
+	lock := broker.topicLocks.RequestSubscriberLock(tp)
+	defer broker.topicLocks.ReleaseLock(tp, false)
+	cond := sync.NewCond(&lock.Mutex)
+
+	lastReadTime := time.Now()
+	switch in.Init.StartPosition {
+	case messaging_pb.SubscriberMessage_InitMessage_TIMESTAMP:
+		lastReadTime = time.Unix(0, in.Init.TimestampNs)
+	case messaging_pb.SubscriberMessage_InitMessage_LATEST:
+	case messaging_pb.SubscriberMessage_InitMessage_EARLIEST:
+	}
+
+	// how to process each message
+	// an error returned will end the subscription
+	eachMessageFn := func(m *messaging_pb.Message) error {
+		err := stream.Send(&messaging_pb.BrokerMessage{
+			Data:     m,
+		})
+		if err != nil {
+			glog.V(0).Infof("=> subscriber %v: %+v", subscriberId, err)
+		}
+		return err
+	}
+
+	// loop through all messages
+	for {
+
+		_, buf := lock.logBuffer.ReadFromBuffer(lastReadTime)
+
+		for pos := 0; pos+4 < len(buf); {
+
+			size := util.BytesToUint32(buf[pos : pos+4])
+			entryData := buf[pos+4 : pos+4+int(size)]
+
+			m := &messaging_pb.Message{}
+			if err = proto.Unmarshal(entryData, m); err != nil {
+				glog.Errorf("unexpected unmarshal messaging_pb.Message: %v", err)
+				pos += 4 + int(size)
+				continue
+			}
+
+			if err = eachMessageFn(m); err != nil {
+				return err
+			}
+
+			lastReadTime = time.Unix(0, m.Timestamp)
+			pos += 4 + int(size)
+		}
+
+		lock.Mutex.Lock()
+		cond.Wait()
+		lock.Mutex.Unlock()
+	}
+
+}
diff --git a/weed/messaging/broker_server.go b/weed/messaging/broker/broker_server.go
similarity index 97%
rename from weed/messaging/broker_server.go
rename to weed/messaging/broker/broker_server.go
index bc842eeea..0522eb4b7 100644
--- a/weed/messaging/broker_server.go
+++ b/weed/messaging/broker/broker_server.go
@@ -1,4 +1,4 @@
-package messaging
+package broker
 
 import (
 	"context"
@@ -23,6 +23,7 @@ type MessageBrokerOption struct {
 type MessageBroker struct {
 	option         *MessageBrokerOption
 	grpcDialOption grpc.DialOption
+	topicLocks     *TopicLocks
 }
 
 func NewMessageBroker(option *MessageBrokerOption, grpcDialOption grpc.DialOption) (messageBroker *MessageBroker, err error) {
@@ -30,6 +31,7 @@ func NewMessageBroker(option *MessageBrokerOption, grpcDialOption grpc.DialOptio
 	messageBroker = &MessageBroker{
 		option:         option,
 		grpcDialOption: grpcDialOption,
+		topicLocks:     NewTopicLocks(),
 	}
 
 	go messageBroker.loopForEver()
diff --git a/weed/messaging/broker/topic_lock.go b/weed/messaging/broker/topic_lock.go
new file mode 100644
index 000000000..9e4ea6824
--- /dev/null
+++ b/weed/messaging/broker/topic_lock.go
@@ -0,0 +1,80 @@
+package broker
+
+import (
+	"sync"
+	"time"
+
+	"github.com/chrislusf/seaweedfs/weed/util/log_buffer"
+)
+
+type TopicPartition struct {
+	Namespace string
+	Topic     string
+	Partition int32
+}
+type TopicLock struct {
+	sync.Mutex
+	subscriberCount int
+	publisherCount  int
+	logBuffer       *log_buffer.LogBuffer
+}
+
+type TopicLocks struct {
+	sync.Mutex
+	locks map[TopicPartition]*TopicLock
+}
+
+func NewTopicLocks() *TopicLocks {
+	return &TopicLocks{
+		locks: make(map[TopicPartition]*TopicLock),
+	}
+}
+
+func (tl *TopicLocks) RequestSubscriberLock(partition TopicPartition) *TopicLock {
+	tl.Lock()
+	defer tl.Unlock()
+
+	lock, found := tl.locks[partition]
+	if !found {
+		lock = &TopicLock{}
+		tl.locks[partition] = lock
+	}
+	lock.subscriberCount++
+
+	return lock
+}
+
+func (tl *TopicLocks) RequestPublisherLock(partition TopicPartition, flushFn func(startTime, stopTime time.Time, buf []byte)) *log_buffer.LogBuffer {
+	tl.Lock()
+	defer tl.Unlock()
+
+	lock, found := tl.locks[partition]
+	if !found {
+		lock = &TopicLock{}
+		tl.locks[partition] = lock
+	}
+	lock.publisherCount++
+	cond := sync.NewCond(&lock.Mutex)
+	lock.logBuffer = log_buffer.NewLogBuffer(time.Minute, flushFn, func() {
+		cond.Broadcast()
+	})
+	return lock.logBuffer
+}
+
+func (tl *TopicLocks) ReleaseLock(partition TopicPartition, isPublisher bool) {
+	tl.Lock()
+	defer tl.Unlock()
+
+	lock, found := tl.locks[partition]
+	if !found {
+		return
+	}
+	if isPublisher {
+		lock.publisherCount--
+	} else {
+		lock.subscriberCount--
+	}
+	if lock.subscriberCount <= 0 && lock.publisherCount <= 0 {
+		delete(tl.locks, partition)
+	}
+}
diff --git a/weed/messaging/broker_grpc_server_subscribe.go b/weed/messaging/broker_grpc_server_subscribe.go
deleted file mode 100644
index 137fcac8a..000000000
--- a/weed/messaging/broker_grpc_server_subscribe.go
+++ /dev/null
@@ -1,9 +0,0 @@
-package messaging
-
-import (
-	"github.com/chrislusf/seaweedfs/weed/pb/messaging_pb"
-)
-
-func (broker *MessageBroker) Subscribe(server messaging_pb.SeaweedMessaging_SubscribeServer) error {
-	panic("implement me")
-}