From dd4ef58d9937168007d8501f56dab8347b50c1d6 Mon Sep 17 00:00:00 2001
From: Chris Lu <chris.lu@gmail.com>
Date: Tue, 14 May 2019 23:35:39 -0700
Subject: [PATCH 01/10] skip setting ip address for filer

related to https://github.com/chrislusf/seaweedfs/issues/960
---
 docker/entrypoint.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker/entrypoint.sh b/docker/entrypoint.sh
index 0265a0cb3..42f71a5c3 100755
--- a/docker/entrypoint.sh
+++ b/docker/entrypoint.sh
@@ -29,7 +29,7 @@ case "$1" in
   	;;
 
   'filer')
-  	ARGS="-ip `hostname -i` "
+  	ARGS=""
   	if [ -n "$MASTER_PORT_9333_TCP_ADDR" ] ; then
 		ARGS="$ARGS -master=$MASTER_PORT_9333_TCP_ADDR:$MASTER_PORT_9333_TCP_PORT"
 	fi

From ab22407763f8dc205c31d30b0532583a6e13b5bf Mon Sep 17 00:00:00 2001
From: Chris Lu <chris.lu@gmail.com>
Date: Tue, 14 May 2019 23:43:40 -0700
Subject: [PATCH 02/10] filer ip only use bind ip

---
 weed/command/server.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/weed/command/server.go b/weed/command/server.go
index 5354fab27..e015e8832 100644
--- a/weed/command/server.go
+++ b/weed/command/server.go
@@ -128,7 +128,7 @@ func runServer(cmd *Command, args []string) bool {
 
 	master := *serverIp + ":" + strconv.Itoa(*masterPort)
 	filerOptions.masters = &master
-	filerOptions.ip = serverIp
+	filerOptions.ip = serverBindIp
 	serverOptions.v.ip = serverIp
 	serverOptions.v.bindIp = serverBindIp
 	serverOptions.v.masters = &master

From 939de1e8326d6b7cd62011f84e21b6596aa640e6 Mon Sep 17 00:00:00 2001
From: Chris Lu <chrislusf@users.noreply.github.com>
Date: Wed, 15 May 2019 23:25:59 -0700
Subject: [PATCH 03/10] Update README.md

---
 README.md | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/README.md b/README.md
index 00772e2c0..9e619e825 100644
--- a/README.md
+++ b/README.md
@@ -15,9 +15,6 @@ possible entirely thanks to the support of these awesome [backers](https://githu
 If you'd like to grow SeaweedFS even stronger, please consider joining our
 <a href="https://www.patreon.com/seaweedfs">sponsors on Patreon</a>.
 
-Platinum ($2500/month), Gold ($500/month): put your company logo on the SeaweedFS github page
-Generous Backer($50/month), Backer($10/month): put your name on the SeaweedFS backer page.
-
 Your support will be really appreciated by me and other supporters!
 
 <h3 align="center"><a href="https://www.patreon.com/seaweedfs">Sponsor SeaweedFS via Patreon</a></h3>

From 82b0759493470417c0eb7135ed4a9b0e530fd9a3 Mon Sep 17 00:00:00 2001
From: Chris Lu <chris.lu@gmail.com>
Date: Fri, 17 May 2019 02:03:23 -0700
Subject: [PATCH 04/10] filer: migrating filer store from persisting shorter
 structured file id instead of a string

---
 other/java/client/pom.xml                     |   2 +-
 .../java/seaweedfs/client/FilerClient.java    |  30 +-
 other/java/client/src/main/proto/filer.proto  |  12 +-
 other/java/hdfs/pom.xml                       |   2 +-
 weed/filer2/filer.go                          |   4 +-
 weed/filer2/filerstore.go                     |  62 ++++
 weed/pb/filer.proto                           |  12 +-
 weed/pb/filer_pb/filer.pb.go                  | 287 +++++++++++-------
 weed/pb/filer_pb/filer_pb_helper.go           |  58 ++++
 weed/pb/filer_pb/filer_pb_helper_test.go      |  18 ++
 10 files changed, 361 insertions(+), 126 deletions(-)
 create mode 100644 weed/pb/filer_pb/filer_pb_helper.go
 create mode 100644 weed/pb/filer_pb/filer_pb_helper_test.go

diff --git a/other/java/client/pom.xml b/other/java/client/pom.xml
index 67b338c37..5882c726d 100644
--- a/other/java/client/pom.xml
+++ b/other/java/client/pom.xml
@@ -4,7 +4,7 @@
 
     <groupId>com.github.chrislusf</groupId>
     <artifactId>seaweedfs-client</artifactId>
-    <version>1.0.9</version>
+    <version>1.1.0</version>
 
     <parent>
         <groupId>org.sonatype.oss</groupId>
diff --git a/other/java/client/src/main/java/seaweedfs/client/FilerClient.java b/other/java/client/src/main/java/seaweedfs/client/FilerClient.java
index 562a36894..f4bd0944b 100644
--- a/other/java/client/src/main/java/seaweedfs/client/FilerClient.java
+++ b/other/java/client/src/main/java/seaweedfs/client/FilerClient.java
@@ -173,21 +173,27 @@ public class FilerClient {
     }
 
     public List<FilerProto.Entry> listEntries(String path, String entryPrefix, String lastEntryName, int limit) {
-        return filerGrpcClient.getBlockingStub().listEntries(FilerProto.ListEntriesRequest.newBuilder()
+        List<FilerProto.Entry> entries = filerGrpcClient.getBlockingStub().listEntries(FilerProto.ListEntriesRequest.newBuilder()
                 .setDirectory(path)
                 .setPrefix(entryPrefix)
                 .setStartFromFileName(lastEntryName)
                 .setLimit(limit)
                 .build()).getEntriesList();
+        List<FilerProto.Entry> fixedEntries = new ArrayList<>(entries.size());
+        for (FilerProto.Entry entry : entries) {
+            fixedEntries.add(fixEntryAfterReading(entry));
+        }
+        return fixedEntries;
     }
 
     public FilerProto.Entry lookupEntry(String directory, String entryName) {
         try {
-            return filerGrpcClient.getBlockingStub().lookupDirectoryEntry(
+            FilerProto.Entry entry = filerGrpcClient.getBlockingStub().lookupDirectoryEntry(
                     FilerProto.LookupDirectoryEntryRequest.newBuilder()
                             .setDirectory(directory)
                             .setName(entryName)
                             .build()).getEntry();
+            return fixEntryAfterReading(entry);
         } catch (Exception e) {
             LOG.warn("lookupEntry {}/{}: {}", directory, entryName, e);
             return null;
@@ -251,4 +257,24 @@ public class FilerClient {
         return true;
     }
 
+    private FilerProto.Entry fixEntryAfterReading(FilerProto.Entry entry) {
+        if (entry.getChunksList().size() <= 0) {
+            return entry;
+        }
+        String fileId = entry.getChunks(0).getFileId();
+        if (fileId != null && fileId.length() != 0) {
+            return entry;
+        }
+        FilerProto.Entry.Builder entryBuilder = entry.toBuilder();
+        entryBuilder.clearChunks();
+        for (FilerProto.FileChunk chunk : entry.getChunksList()) {
+            FilerProto.FileChunk.Builder chunkBuilder = chunk.toBuilder();
+            FilerProto.FileId fid = chunk.getFid();
+            fileId = String.format("%d,%d%x", fid.getVolumeId(), fid.getFileKey(), fid.getCookie());
+            chunkBuilder.setFileId(fileId);
+            entryBuilder.addChunks(chunkBuilder);
+        }
+        return entryBuilder.build();
+    }
+
 }
diff --git a/other/java/client/src/main/proto/filer.proto b/other/java/client/src/main/proto/filer.proto
index 350288b53..56814c39a 100644
--- a/other/java/client/src/main/proto/filer.proto
+++ b/other/java/client/src/main/proto/filer.proto
@@ -85,12 +85,20 @@ message EventNotification {
 }
 
 message FileChunk {
-    string file_id = 1;
+    string file_id = 1; // to be deprecated
     int64 offset = 2;
     uint64 size = 3;
     int64 mtime = 4;
     string e_tag = 5;
-    string source_file_id = 6;
+    string source_file_id = 6; // to be deprecated
+    FileId fid = 7;
+    FileId source_fid = 8;
+}
+
+message FileId {
+    uint32 volume_id = 1;
+    uint64 file_key = 2;
+    fixed32 cookie = 3;
 }
 
 message FuseAttributes {
diff --git a/other/java/hdfs/pom.xml b/other/java/hdfs/pom.xml
index 35911d463..6a1cd897f 100644
--- a/other/java/hdfs/pom.xml
+++ b/other/java/hdfs/pom.xml
@@ -5,7 +5,7 @@
     <modelVersion>4.0.0</modelVersion>
 
     <properties>
-        <seaweedfs.client.version>1.0.9</seaweedfs.client.version>
+        <seaweedfs.client.version>1.1.0</seaweedfs.client.version>
         <hadoop.version>3.1.1</hadoop.version>
     </properties>
 
diff --git a/weed/filer2/filer.go b/weed/filer2/filer.go
index 6b784e0b2..68ca3180f 100644
--- a/weed/filer2/filer.go
+++ b/weed/filer2/filer.go
@@ -21,7 +21,7 @@ var (
 )
 
 type Filer struct {
-	store              FilerStore
+	store              *FilerStoreWrapper
 	directoryCache     *ccache.Cache
 	MasterClient       *wdclient.MasterClient
 	fileIdDeletionChan chan string
@@ -42,7 +42,7 @@ func NewFiler(masters []string, grpcDialOption grpc.DialOption) *Filer {
 }
 
 func (f *Filer) SetStore(store FilerStore) {
-	f.store = store
+	f.store = NewFilerStoreWrapper(store)
 }
 
 func (f *Filer) DisableDirectoryCache() {
diff --git a/weed/filer2/filerstore.go b/weed/filer2/filerstore.go
index 0b256e56e..7755ee1a3 100644
--- a/weed/filer2/filerstore.go
+++ b/weed/filer2/filerstore.go
@@ -3,6 +3,7 @@ package filer2
 import (
 	"context"
 	"errors"
+	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
 	"github.com/chrislusf/seaweedfs/weed/util"
 )
 
@@ -24,3 +25,64 @@ type FilerStore interface {
 }
 
 var ErrNotFound = errors.New("filer: no entry is found in filer store")
+
+type FilerStoreWrapper struct {
+	actualStore FilerStore
+}
+
+func NewFilerStoreWrapper(store FilerStore) *FilerStoreWrapper{
+	return &FilerStoreWrapper{
+		actualStore:store,
+	}
+}
+
+func (fsw *FilerStoreWrapper) GetName() string {
+	return fsw.actualStore.GetName()
+}
+
+func (fsw *FilerStoreWrapper) Initialize(configuration util.Configuration) error {
+	return fsw.actualStore.Initialize(configuration)
+}
+
+func (fsw *FilerStoreWrapper) InsertEntry(ctx context.Context, entry *Entry) error {
+	filer_pb.BeforeEntrySerialization(entry.Chunks)
+	return fsw.actualStore.InsertEntry(ctx, entry)
+}
+
+func (fsw *FilerStoreWrapper) UpdateEntry(ctx context.Context, entry *Entry) error {
+	filer_pb.BeforeEntrySerialization(entry.Chunks)
+	return fsw.actualStore.UpdateEntry(ctx, entry)
+}
+
+func (fsw *FilerStoreWrapper) FindEntry(ctx context.Context, fp FullPath) (entry *Entry, err error) {
+	entry, err = fsw.actualStore.FindEntry(ctx, fp)
+	filer_pb.AfterEntryDeserialization(entry.Chunks)
+	return
+}
+
+func (fsw *FilerStoreWrapper) DeleteEntry(ctx context.Context, fp FullPath) (err error) {
+	return fsw.actualStore.DeleteEntry(ctx, fp)
+}
+
+func (fsw *FilerStoreWrapper) ListDirectoryEntries(ctx context.Context, dirPath FullPath, startFileName string, includeStartFile bool, limit int) ([]*Entry, error) {
+	entries, err := fsw.actualStore.ListDirectoryEntries(ctx, dirPath, startFileName, includeStartFile, limit)
+	if err != nil {
+		return nil, err
+	}
+	for _, entry := range entries {
+		filer_pb.AfterEntryDeserialization(entry.Chunks)
+	}
+	return entries, err
+}
+
+func (fsw *FilerStoreWrapper) BeginTransaction(ctx context.Context) (context.Context, error) {
+	return fsw.actualStore.BeginTransaction(ctx)
+}
+
+func (fsw *FilerStoreWrapper) CommitTransaction(ctx context.Context) error {
+	return fsw.actualStore.CommitTransaction(ctx)
+}
+
+func (fsw *FilerStoreWrapper) RollbackTransaction(ctx context.Context) error {
+	return fsw.actualStore.RollbackTransaction(ctx)
+}
diff --git a/weed/pb/filer.proto b/weed/pb/filer.proto
index 350288b53..56814c39a 100644
--- a/weed/pb/filer.proto
+++ b/weed/pb/filer.proto
@@ -85,12 +85,20 @@ message EventNotification {
 }
 
 message FileChunk {
-    string file_id = 1;
+    string file_id = 1; // to be deprecated
     int64 offset = 2;
     uint64 size = 3;
     int64 mtime = 4;
     string e_tag = 5;
-    string source_file_id = 6;
+    string source_file_id = 6; // to be deprecated
+    FileId fid = 7;
+    FileId source_fid = 8;
+}
+
+message FileId {
+    uint32 volume_id = 1;
+    uint64 file_key = 2;
+    fixed32 cookie = 3;
 }
 
 message FuseAttributes {
diff --git a/weed/pb/filer_pb/filer.pb.go b/weed/pb/filer_pb/filer.pb.go
index 3e717f0e6..3d39af65c 100644
--- a/weed/pb/filer_pb/filer.pb.go
+++ b/weed/pb/filer_pb/filer.pb.go
@@ -17,6 +17,7 @@ It has these top-level messages:
 	FullEntry
 	EventNotification
 	FileChunk
+	FileId
 	FuseAttributes
 	CreateEntryRequest
 	CreateEntryResponse
@@ -276,12 +277,14 @@ func (m *EventNotification) GetNewParentPath() string {
 }
 
 type FileChunk struct {
-	FileId       string `protobuf:"bytes,1,opt,name=file_id,json=fileId" json:"file_id,omitempty"`
-	Offset       int64  `protobuf:"varint,2,opt,name=offset" json:"offset,omitempty"`
-	Size         uint64 `protobuf:"varint,3,opt,name=size" json:"size,omitempty"`
-	Mtime        int64  `protobuf:"varint,4,opt,name=mtime" json:"mtime,omitempty"`
-	ETag         string `protobuf:"bytes,5,opt,name=e_tag,json=eTag" json:"e_tag,omitempty"`
-	SourceFileId string `protobuf:"bytes,6,opt,name=source_file_id,json=sourceFileId" json:"source_file_id,omitempty"`
+	FileId       string  `protobuf:"bytes,1,opt,name=file_id,json=fileId" json:"file_id,omitempty"`
+	Offset       int64   `protobuf:"varint,2,opt,name=offset" json:"offset,omitempty"`
+	Size         uint64  `protobuf:"varint,3,opt,name=size" json:"size,omitempty"`
+	Mtime        int64   `protobuf:"varint,4,opt,name=mtime" json:"mtime,omitempty"`
+	ETag         string  `protobuf:"bytes,5,opt,name=e_tag,json=eTag" json:"e_tag,omitempty"`
+	SourceFileId string  `protobuf:"bytes,6,opt,name=source_file_id,json=sourceFileId" json:"source_file_id,omitempty"`
+	Fid          *FileId `protobuf:"bytes,7,opt,name=fid" json:"fid,omitempty"`
+	SourceFid    *FileId `protobuf:"bytes,8,opt,name=source_fid,json=sourceFid" json:"source_fid,omitempty"`
 }
 
 func (m *FileChunk) Reset()                    { *m = FileChunk{} }
@@ -331,6 +334,52 @@ func (m *FileChunk) GetSourceFileId() string {
 	return ""
 }
 
+func (m *FileChunk) GetFid() *FileId {
+	if m != nil {
+		return m.Fid
+	}
+	return nil
+}
+
+func (m *FileChunk) GetSourceFid() *FileId {
+	if m != nil {
+		return m.SourceFid
+	}
+	return nil
+}
+
+type FileId struct {
+	VolumeId uint32 `protobuf:"varint,1,opt,name=volume_id,json=volumeId" json:"volume_id,omitempty"`
+	FileKey  uint64 `protobuf:"varint,2,opt,name=file_key,json=fileKey" json:"file_key,omitempty"`
+	Cookie   uint32 `protobuf:"fixed32,3,opt,name=cookie" json:"cookie,omitempty"`
+}
+
+func (m *FileId) Reset()                    { *m = FileId{} }
+func (m *FileId) String() string            { return proto.CompactTextString(m) }
+func (*FileId) ProtoMessage()               {}
+func (*FileId) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{8} }
+
+func (m *FileId) GetVolumeId() uint32 {
+	if m != nil {
+		return m.VolumeId
+	}
+	return 0
+}
+
+func (m *FileId) GetFileKey() uint64 {
+	if m != nil {
+		return m.FileKey
+	}
+	return 0
+}
+
+func (m *FileId) GetCookie() uint32 {
+	if m != nil {
+		return m.Cookie
+	}
+	return 0
+}
+
 type FuseAttributes struct {
 	FileSize      uint64   `protobuf:"varint,1,opt,name=file_size,json=fileSize" json:"file_size,omitempty"`
 	Mtime         int64    `protobuf:"varint,2,opt,name=mtime" json:"mtime,omitempty"`
@@ -350,7 +399,7 @@ type FuseAttributes struct {
 func (m *FuseAttributes) Reset()                    { *m = FuseAttributes{} }
 func (m *FuseAttributes) String() string            { return proto.CompactTextString(m) }
 func (*FuseAttributes) ProtoMessage()               {}
-func (*FuseAttributes) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{8} }
+func (*FuseAttributes) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{9} }
 
 func (m *FuseAttributes) GetFileSize() uint64 {
 	if m != nil {
@@ -451,7 +500,7 @@ type CreateEntryRequest struct {
 func (m *CreateEntryRequest) Reset()                    { *m = CreateEntryRequest{} }
 func (m *CreateEntryRequest) String() string            { return proto.CompactTextString(m) }
 func (*CreateEntryRequest) ProtoMessage()               {}
-func (*CreateEntryRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{9} }
+func (*CreateEntryRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{10} }
 
 func (m *CreateEntryRequest) GetDirectory() string {
 	if m != nil {
@@ -473,7 +522,7 @@ type CreateEntryResponse struct {
 func (m *CreateEntryResponse) Reset()                    { *m = CreateEntryResponse{} }
 func (m *CreateEntryResponse) String() string            { return proto.CompactTextString(m) }
 func (*CreateEntryResponse) ProtoMessage()               {}
-func (*CreateEntryResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{10} }
+func (*CreateEntryResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{11} }
 
 type UpdateEntryRequest struct {
 	Directory string `protobuf:"bytes,1,opt,name=directory" json:"directory,omitempty"`
@@ -483,7 +532,7 @@ type UpdateEntryRequest struct {
 func (m *UpdateEntryRequest) Reset()                    { *m = UpdateEntryRequest{} }
 func (m *UpdateEntryRequest) String() string            { return proto.CompactTextString(m) }
 func (*UpdateEntryRequest) ProtoMessage()               {}
-func (*UpdateEntryRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{11} }
+func (*UpdateEntryRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{12} }
 
 func (m *UpdateEntryRequest) GetDirectory() string {
 	if m != nil {
@@ -505,7 +554,7 @@ type UpdateEntryResponse struct {
 func (m *UpdateEntryResponse) Reset()                    { *m = UpdateEntryResponse{} }
 func (m *UpdateEntryResponse) String() string            { return proto.CompactTextString(m) }
 func (*UpdateEntryResponse) ProtoMessage()               {}
-func (*UpdateEntryResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{12} }
+func (*UpdateEntryResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{13} }
 
 type DeleteEntryRequest struct {
 	Directory string `protobuf:"bytes,1,opt,name=directory" json:"directory,omitempty"`
@@ -518,7 +567,7 @@ type DeleteEntryRequest struct {
 func (m *DeleteEntryRequest) Reset()                    { *m = DeleteEntryRequest{} }
 func (m *DeleteEntryRequest) String() string            { return proto.CompactTextString(m) }
 func (*DeleteEntryRequest) ProtoMessage()               {}
-func (*DeleteEntryRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{13} }
+func (*DeleteEntryRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{14} }
 
 func (m *DeleteEntryRequest) GetDirectory() string {
 	if m != nil {
@@ -554,7 +603,7 @@ type DeleteEntryResponse struct {
 func (m *DeleteEntryResponse) Reset()                    { *m = DeleteEntryResponse{} }
 func (m *DeleteEntryResponse) String() string            { return proto.CompactTextString(m) }
 func (*DeleteEntryResponse) ProtoMessage()               {}
-func (*DeleteEntryResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{14} }
+func (*DeleteEntryResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{15} }
 
 type AtomicRenameEntryRequest struct {
 	OldDirectory string `protobuf:"bytes,1,opt,name=old_directory,json=oldDirectory" json:"old_directory,omitempty"`
@@ -566,7 +615,7 @@ type AtomicRenameEntryRequest struct {
 func (m *AtomicRenameEntryRequest) Reset()                    { *m = AtomicRenameEntryRequest{} }
 func (m *AtomicRenameEntryRequest) String() string            { return proto.CompactTextString(m) }
 func (*AtomicRenameEntryRequest) ProtoMessage()               {}
-func (*AtomicRenameEntryRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{15} }
+func (*AtomicRenameEntryRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{16} }
 
 func (m *AtomicRenameEntryRequest) GetOldDirectory() string {
 	if m != nil {
@@ -602,7 +651,7 @@ type AtomicRenameEntryResponse struct {
 func (m *AtomicRenameEntryResponse) Reset()                    { *m = AtomicRenameEntryResponse{} }
 func (m *AtomicRenameEntryResponse) String() string            { return proto.CompactTextString(m) }
 func (*AtomicRenameEntryResponse) ProtoMessage()               {}
-func (*AtomicRenameEntryResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{16} }
+func (*AtomicRenameEntryResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{17} }
 
 type AssignVolumeRequest struct {
 	Count       int32  `protobuf:"varint,1,opt,name=count" json:"count,omitempty"`
@@ -615,7 +664,7 @@ type AssignVolumeRequest struct {
 func (m *AssignVolumeRequest) Reset()                    { *m = AssignVolumeRequest{} }
 func (m *AssignVolumeRequest) String() string            { return proto.CompactTextString(m) }
 func (*AssignVolumeRequest) ProtoMessage()               {}
-func (*AssignVolumeRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{17} }
+func (*AssignVolumeRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{18} }
 
 func (m *AssignVolumeRequest) GetCount() int32 {
 	if m != nil {
@@ -663,7 +712,7 @@ type AssignVolumeResponse struct {
 func (m *AssignVolumeResponse) Reset()                    { *m = AssignVolumeResponse{} }
 func (m *AssignVolumeResponse) String() string            { return proto.CompactTextString(m) }
 func (*AssignVolumeResponse) ProtoMessage()               {}
-func (*AssignVolumeResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{18} }
+func (*AssignVolumeResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{19} }
 
 func (m *AssignVolumeResponse) GetFileId() string {
 	if m != nil {
@@ -707,7 +756,7 @@ type LookupVolumeRequest struct {
 func (m *LookupVolumeRequest) Reset()                    { *m = LookupVolumeRequest{} }
 func (m *LookupVolumeRequest) String() string            { return proto.CompactTextString(m) }
 func (*LookupVolumeRequest) ProtoMessage()               {}
-func (*LookupVolumeRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{19} }
+func (*LookupVolumeRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{20} }
 
 func (m *LookupVolumeRequest) GetVolumeIds() []string {
 	if m != nil {
@@ -723,7 +772,7 @@ type Locations struct {
 func (m *Locations) Reset()                    { *m = Locations{} }
 func (m *Locations) String() string            { return proto.CompactTextString(m) }
 func (*Locations) ProtoMessage()               {}
-func (*Locations) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{20} }
+func (*Locations) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{21} }
 
 func (m *Locations) GetLocations() []*Location {
 	if m != nil {
@@ -740,7 +789,7 @@ type Location struct {
 func (m *Location) Reset()                    { *m = Location{} }
 func (m *Location) String() string            { return proto.CompactTextString(m) }
 func (*Location) ProtoMessage()               {}
-func (*Location) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{21} }
+func (*Location) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{22} }
 
 func (m *Location) GetUrl() string {
 	if m != nil {
@@ -763,7 +812,7 @@ type LookupVolumeResponse struct {
 func (m *LookupVolumeResponse) Reset()                    { *m = LookupVolumeResponse{} }
 func (m *LookupVolumeResponse) String() string            { return proto.CompactTextString(m) }
 func (*LookupVolumeResponse) ProtoMessage()               {}
-func (*LookupVolumeResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{22} }
+func (*LookupVolumeResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{23} }
 
 func (m *LookupVolumeResponse) GetLocationsMap() map[string]*Locations {
 	if m != nil {
@@ -779,7 +828,7 @@ type DeleteCollectionRequest struct {
 func (m *DeleteCollectionRequest) Reset()                    { *m = DeleteCollectionRequest{} }
 func (m *DeleteCollectionRequest) String() string            { return proto.CompactTextString(m) }
 func (*DeleteCollectionRequest) ProtoMessage()               {}
-func (*DeleteCollectionRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{23} }
+func (*DeleteCollectionRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{24} }
 
 func (m *DeleteCollectionRequest) GetCollection() string {
 	if m != nil {
@@ -794,7 +843,7 @@ type DeleteCollectionResponse struct {
 func (m *DeleteCollectionResponse) Reset()                    { *m = DeleteCollectionResponse{} }
 func (m *DeleteCollectionResponse) String() string            { return proto.CompactTextString(m) }
 func (*DeleteCollectionResponse) ProtoMessage()               {}
-func (*DeleteCollectionResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{24} }
+func (*DeleteCollectionResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{25} }
 
 type StatisticsRequest struct {
 	Replication string `protobuf:"bytes,1,opt,name=replication" json:"replication,omitempty"`
@@ -805,7 +854,7 @@ type StatisticsRequest struct {
 func (m *StatisticsRequest) Reset()                    { *m = StatisticsRequest{} }
 func (m *StatisticsRequest) String() string            { return proto.CompactTextString(m) }
 func (*StatisticsRequest) ProtoMessage()               {}
-func (*StatisticsRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{25} }
+func (*StatisticsRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{26} }
 
 func (m *StatisticsRequest) GetReplication() string {
 	if m != nil {
@@ -840,7 +889,7 @@ type StatisticsResponse struct {
 func (m *StatisticsResponse) Reset()                    { *m = StatisticsResponse{} }
 func (m *StatisticsResponse) String() string            { return proto.CompactTextString(m) }
 func (*StatisticsResponse) ProtoMessage()               {}
-func (*StatisticsResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{26} }
+func (*StatisticsResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{27} }
 
 func (m *StatisticsResponse) GetReplication() string {
 	if m != nil {
@@ -893,6 +942,7 @@ func init() {
 	proto.RegisterType((*FullEntry)(nil), "filer_pb.FullEntry")
 	proto.RegisterType((*EventNotification)(nil), "filer_pb.EventNotification")
 	proto.RegisterType((*FileChunk)(nil), "filer_pb.FileChunk")
+	proto.RegisterType((*FileId)(nil), "filer_pb.FileId")
 	proto.RegisterType((*FuseAttributes)(nil), "filer_pb.FuseAttributes")
 	proto.RegisterType((*CreateEntryRequest)(nil), "filer_pb.CreateEntryRequest")
 	proto.RegisterType((*CreateEntryResponse)(nil), "filer_pb.CreateEntryResponse")
@@ -1286,95 +1336,100 @@ var _SeaweedFiler_serviceDesc = grpc.ServiceDesc{
 func init() { proto.RegisterFile("filer.proto", fileDescriptor0) }
 
 var fileDescriptor0 = []byte{
-	// 1435 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xb4, 0x57, 0xdd, 0x6e, 0xdc, 0x44,
-	0x14, 0xae, 0xf7, 0x2f, 0xeb, 0xb3, 0xbb, 0x6d, 0x32, 0x09, 0x74, 0xbb, 0x49, 0xca, 0xd6, 0xa1,
-	0x55, 0x2a, 0xaa, 0xa8, 0x2a, 0x5c, 0xb4, 0x54, 0x48, 0xb4, 0xf9, 0x91, 0x2a, 0xa5, 0x3f, 0x72,
-	0x5a, 0x24, 0x84, 0x84, 0xe5, 0xd8, 0x93, 0xcd, 0x28, 0xb3, 0xf6, 0xe2, 0x19, 0x27, 0x0d, 0x8f,
-	0xc0, 0x0d, 0xf7, 0x48, 0x5c, 0x70, 0xc5, 0x4b, 0x20, 0x6e, 0x78, 0x0a, 0x5e, 0x82, 0x67, 0x40,
-	0x67, 0xc6, 0xf6, 0x8e, 0xd7, 0x9b, 0xb4, 0x08, 0xf5, 0x6e, 0xe6, 0x3b, 0x73, 0xce, 0xf9, 0xe6,
-	0xcc, 0xf9, 0xb1, 0xa1, 0x73, 0xc4, 0x38, 0x4d, 0xb6, 0x26, 0x49, 0x2c, 0x63, 0xd2, 0x56, 0x1b,
-	0x6f, 0x72, 0xe8, 0xbc, 0x84, 0xd5, 0xfd, 0x38, 0x3e, 0x49, 0x27, 0x3b, 0x2c, 0xa1, 0x81, 0x8c,
-	0x93, 0xf3, 0xdd, 0x48, 0x26, 0xe7, 0x2e, 0xfd, 0x21, 0xa5, 0x42, 0x92, 0x35, 0xb0, 0xc3, 0x5c,
-	0xd0, 0xb7, 0x86, 0xd6, 0xa6, 0xed, 0x4e, 0x01, 0x42, 0xa0, 0x11, 0xf9, 0x63, 0xda, 0xaf, 0x29,
-	0x81, 0x5a, 0x3b, 0xbb, 0xb0, 0x36, 0xdf, 0xa0, 0x98, 0xc4, 0x91, 0xa0, 0xe4, 0x36, 0x34, 0x29,
-	0x02, 0xca, 0x5a, 0xe7, 0xc1, 0xb5, 0xad, 0x9c, 0xca, 0x96, 0x3e, 0xa7, 0xa5, 0xce, 0x9f, 0x16,
-	0x90, 0x7d, 0x26, 0x24, 0x82, 0x8c, 0x8a, 0xf7, 0xe3, 0xf3, 0x31, 0xb4, 0x26, 0x09, 0x3d, 0x62,
-	0x6f, 0x33, 0x46, 0xd9, 0x8e, 0xdc, 0x83, 0x25, 0x21, 0xfd, 0x44, 0xee, 0x25, 0xf1, 0x78, 0x8f,
-	0x71, 0xfa, 0x02, 0x49, 0xd7, 0xd5, 0x91, 0xaa, 0x80, 0x6c, 0x01, 0x61, 0x51, 0xc0, 0x53, 0xc1,
-	0x4e, 0xe9, 0x41, 0x2e, 0xed, 0x37, 0x86, 0xd6, 0x66, 0xdb, 0x9d, 0x23, 0x21, 0x2b, 0xd0, 0xe4,
-	0x6c, 0xcc, 0x64, 0xbf, 0x39, 0xb4, 0x36, 0x7b, 0xae, 0xde, 0x38, 0x5f, 0xc3, 0x72, 0x89, 0x7f,
-	0x76, 0xfd, 0xbb, 0xb0, 0x40, 0x35, 0xd4, 0xb7, 0x86, 0xf5, 0x79, 0x01, 0xc8, 0xe5, 0xce, 0xaf,
-	0x35, 0x68, 0x2a, 0xa8, 0x88, 0xb3, 0x35, 0x8d, 0x33, 0xb9, 0x05, 0x5d, 0x26, 0xbc, 0x69, 0x30,
-	0x6a, 0x8a, 0x5f, 0x87, 0x89, 0x22, 0xee, 0xe4, 0x33, 0x68, 0x05, 0xc7, 0x69, 0x74, 0x22, 0xfa,
-	0x75, 0xe5, 0x6a, 0x79, 0xea, 0x0a, 0x2f, 0xbb, 0x8d, 0x32, 0x37, 0x3b, 0x42, 0x1e, 0x02, 0xf8,
-	0x52, 0x26, 0xec, 0x30, 0x95, 0x54, 0xa8, 0xdb, 0x76, 0x1e, 0xf4, 0x0d, 0x85, 0x54, 0xd0, 0x27,
-	0x85, 0xdc, 0x35, 0xce, 0x92, 0x47, 0xd0, 0xa6, 0x6f, 0x25, 0x8d, 0x42, 0x1a, 0xf6, 0x9b, 0xca,
-	0xd1, 0xfa, 0xcc, 0x9d, 0xb6, 0x76, 0x33, 0xb9, 0xbe, 0x61, 0x71, 0x7c, 0xf0, 0x18, 0x7a, 0x25,
-	0x11, 0x59, 0x84, 0xfa, 0x09, 0xcd, 0x5f, 0x16, 0x97, 0x18, 0xdd, 0x53, 0x9f, 0xa7, 0x3a, 0xc9,
-	0xba, 0xae, 0xde, 0x7c, 0x59, 0x7b, 0x68, 0x39, 0x3b, 0x60, 0xef, 0xa5, 0x9c, 0x17, 0x8a, 0x21,
-	0x4b, 0x72, 0xc5, 0x90, 0x25, 0xd3, 0x44, 0xab, 0x5d, 0x9a, 0x68, 0x7f, 0x58, 0xb0, 0xb4, 0x7b,
-	0x4a, 0x23, 0xf9, 0x22, 0x96, 0xec, 0x88, 0x05, 0xbe, 0x64, 0x71, 0x44, 0xee, 0x81, 0x1d, 0xf3,
-	0xd0, 0xbb, 0x34, 0x53, 0xdb, 0x31, 0xcf, 0x58, 0xdf, 0x03, 0x3b, 0xa2, 0x67, 0xde, 0xa5, 0xee,
-	0xda, 0x11, 0x3d, 0xd3, 0xa7, 0x37, 0xa0, 0x17, 0x52, 0x4e, 0x25, 0xf5, 0x8a, 0xd7, 0xc1, 0xa7,
-	0xeb, 0x6a, 0x70, 0x5b, 0x3f, 0xc7, 0x1d, 0xb8, 0x86, 0x26, 0x27, 0x7e, 0x42, 0x23, 0xe9, 0x4d,
-	0x7c, 0x79, 0xac, 0xde, 0xc4, 0x76, 0x7b, 0x11, 0x3d, 0x7b, 0xa5, 0xd0, 0x57, 0xbe, 0x3c, 0x76,
-	0x7e, 0xb3, 0xc0, 0x2e, 0x1e, 0x93, 0x5c, 0x87, 0x05, 0x74, 0xeb, 0xb1, 0x30, 0x8b, 0x44, 0x0b,
-	0xb7, 0xcf, 0x42, 0xac, 0x8c, 0xf8, 0xe8, 0x48, 0x50, 0xa9, 0xe8, 0xd5, 0xdd, 0x6c, 0x87, 0x99,
-	0x25, 0xd8, 0x8f, 0xba, 0x18, 0x1a, 0xae, 0x5a, 0x63, 0xc4, 0xc7, 0x92, 0x8d, 0xa9, 0x72, 0x58,
-	0x77, 0xf5, 0x86, 0x2c, 0x43, 0x93, 0x7a, 0xd2, 0x1f, 0xa9, 0x2c, 0xb7, 0xdd, 0x06, 0x7d, 0xed,
-	0x8f, 0xc8, 0xa7, 0x70, 0x55, 0xc4, 0x69, 0x12, 0x50, 0x2f, 0x77, 0xdb, 0x52, 0xd2, 0xae, 0x46,
-	0xf7, 0x94, 0x73, 0xe7, 0x9f, 0x1a, 0x5c, 0x2d, 0xe7, 0x0f, 0x59, 0x05, 0x5b, 0x69, 0x28, 0xe7,
-	0x96, 0x72, 0xae, 0x7a, 0xd2, 0x41, 0x89, 0x40, 0xcd, 0x24, 0x90, 0xab, 0x8c, 0xe3, 0x50, 0xf3,
-	0xed, 0x69, 0x95, 0xe7, 0x71, 0x48, 0xf1, 0xf9, 0x53, 0x16, 0x2a, 0xc6, 0x3d, 0x17, 0x97, 0x88,
-	0x8c, 0x58, 0x98, 0xd5, 0x24, 0x2e, 0x31, 0x06, 0x41, 0xa2, 0xec, 0xb6, 0x74, 0x0c, 0xf4, 0x0e,
-	0x63, 0x30, 0x46, 0x74, 0x41, 0x5f, 0x0c, 0xd7, 0x64, 0x08, 0x9d, 0x84, 0x4e, 0x78, 0x96, 0x0e,
-	0xfd, 0xb6, 0x12, 0x99, 0x10, 0xb9, 0x09, 0x10, 0xc4, 0x9c, 0xd3, 0x40, 0x1d, 0xb0, 0xd5, 0x01,
-	0x03, 0xc1, 0xa7, 0x90, 0x92, 0x7b, 0x82, 0x06, 0x7d, 0x18, 0x5a, 0x9b, 0x4d, 0xb7, 0x25, 0x25,
-	0x3f, 0xa0, 0x01, 0xde, 0x23, 0x15, 0x34, 0xf1, 0x54, 0x45, 0x77, 0x94, 0x5e, 0x1b, 0x01, 0xd5,
-	0x7b, 0xd6, 0x01, 0x46, 0x49, 0x9c, 0x4e, 0xb4, 0xb4, 0x3b, 0xac, 0x63, 0x83, 0x53, 0x88, 0x12,
-	0xdf, 0x86, 0xab, 0xe2, 0x7c, 0xcc, 0x59, 0x74, 0xe2, 0x49, 0x3f, 0x19, 0x51, 0xd9, 0xef, 0xe9,
-	0xa4, 0xc8, 0xd0, 0xd7, 0x0a, 0x74, 0xbe, 0x05, 0xb2, 0x9d, 0x50, 0x5f, 0xd2, 0xff, 0xd0, 0xcb,
-	0xdf, 0xb3, 0x5c, 0x3e, 0x82, 0xe5, 0x92, 0x69, 0xdd, 0xd6, 0xd0, 0xe3, 0x9b, 0x49, 0xf8, 0xa1,
-	0x3c, 0x96, 0x4c, 0x67, 0x1e, 0x7f, 0xb6, 0x80, 0xec, 0xa8, 0x8a, 0xf9, 0x7f, 0x03, 0x0b, 0x73,
-	0x18, 0x1b, 0xa9, 0xae, 0xc8, 0xd0, 0x97, 0x7e, 0xd6, 0xea, 0xbb, 0x4c, 0x68, 0xfb, 0x3b, 0xbe,
-	0xf4, 0xb3, 0x76, 0x9b, 0xd0, 0x20, 0x4d, 0xb0, 0xfb, 0xab, 0xbc, 0x52, 0xed, 0xd6, 0xcd, 0x21,
-	0x24, 0x5a, 0x22, 0x94, 0x11, 0xfd, 0xc5, 0x82, 0xfe, 0x13, 0x19, 0x8f, 0x59, 0xe0, 0x52, 0x74,
-	0x58, 0xa2, 0xbb, 0x01, 0x3d, 0xec, 0x33, 0xb3, 0x94, 0xbb, 0x31, 0x0f, 0xa7, 0x7d, 0xfc, 0x06,
-	0x60, 0xab, 0xf1, 0x0c, 0xe6, 0x0b, 0x31, 0x0f, 0x55, 0x42, 0x6c, 0x00, 0xf6, 0x03, 0x43, 0x5f,
-	0x4f, 0xb5, 0x6e, 0x44, 0xcf, 0x4a, 0xfa, 0x78, 0x48, 0xe9, 0xeb, 0x26, 0xb2, 0x10, 0xd1, 0x33,
-	0xd4, 0x77, 0x56, 0xe1, 0xc6, 0x1c, 0x6e, 0x19, 0xf3, 0xdf, 0x2d, 0x58, 0x7e, 0x22, 0x04, 0x1b,
-	0x45, 0xdf, 0xc4, 0x3c, 0x1d, 0xd3, 0x9c, 0xf4, 0x0a, 0x34, 0x83, 0x38, 0x8d, 0xa4, 0x22, 0xdb,
-	0x74, 0xf5, 0x66, 0xa6, 0x20, 0x6a, 0x95, 0x82, 0x98, 0x29, 0xa9, 0x7a, 0xb5, 0xa4, 0x8c, 0x92,
-	0x69, 0x94, 0x4a, 0xe6, 0x13, 0xe8, 0xe0, 0xc3, 0x78, 0x01, 0x8d, 0x24, 0x4d, 0xb2, 0x0e, 0x04,
-	0x08, 0x6d, 0x2b, 0xc4, 0xf9, 0xc9, 0x82, 0x95, 0x32, 0xd3, 0x6c, 0xdc, 0x5e, 0xd8, 0x10, 0xb1,
-	0x61, 0x24, 0x3c, 0xa3, 0x89, 0x4b, 0x2c, 0xbd, 0x49, 0x7a, 0xc8, 0x59, 0xe0, 0xa1, 0x40, 0xd3,
-	0xb3, 0x35, 0xf2, 0x26, 0xe1, 0xd3, 0x4b, 0x37, 0xcc, 0x4b, 0x13, 0x68, 0xf8, 0xa9, 0x3c, 0xce,
-	0x9b, 0x22, 0xae, 0x9d, 0x2f, 0x60, 0x59, 0x7f, 0x01, 0x95, 0xa3, 0xb6, 0x0e, 0x70, 0xaa, 0x00,
-	0x8f, 0x85, 0x7a, 0xf8, 0xdb, 0xae, 0xad, 0x91, 0x67, 0xa1, 0x70, 0xbe, 0x02, 0x7b, 0x3f, 0xd6,
-	0x81, 0x10, 0xe4, 0x3e, 0xd8, 0x3c, 0xdf, 0x64, 0xdf, 0x09, 0x64, 0x5a, 0x1e, 0xf9, 0x39, 0x77,
-	0x7a, 0xc8, 0x79, 0x0c, 0xed, 0x1c, 0xce, 0xef, 0x66, 0x5d, 0x74, 0xb7, 0xda, 0xcc, 0xdd, 0x9c,
-	0xbf, 0x2c, 0x58, 0x29, 0x53, 0xce, 0xc2, 0xf7, 0x06, 0x7a, 0x85, 0x0b, 0x6f, 0xec, 0x4f, 0x32,
-	0x2e, 0xf7, 0x4d, 0x2e, 0x55, 0xb5, 0x82, 0xa0, 0x78, 0xee, 0x4f, 0x74, 0x4a, 0x75, 0xb9, 0x01,
-	0x0d, 0x5e, 0xc3, 0x52, 0xe5, 0xc8, 0x9c, 0xd1, 0x7f, 0xd7, 0x1c, 0xfd, 0xa5, 0xcf, 0x97, 0x42,
-	0xdb, 0xfc, 0x1e, 0x78, 0x04, 0xd7, 0x75, 0xfd, 0x6d, 0x17, 0x49, 0x97, 0xc7, 0xbe, 0x9c, 0x9b,
-	0xd6, 0x6c, 0x6e, 0x3a, 0x03, 0xe8, 0x57, 0x55, 0xb3, 0x2a, 0x18, 0xc1, 0xd2, 0x81, 0xf4, 0x25,
-	0x13, 0x92, 0x05, 0xc5, 0x77, 0xe8, 0x4c, 0x32, 0x5b, 0xef, 0x9a, 0x0f, 0xd5, 0x72, 0x58, 0x84,
-	0xba, 0x94, 0x79, 0x9e, 0xe1, 0x12, 0x5f, 0x81, 0x98, 0x9e, 0xb2, 0x37, 0xf8, 0x00, 0xae, 0x30,
-	0x1f, 0x64, 0x2c, 0x7d, 0xae, 0xe7, 0x6f, 0x43, 0xcd, 0x5f, 0x5b, 0x21, 0x6a, 0x00, 0xeb, 0x11,
-	0x15, 0x6a, 0x69, 0x53, 0x4f, 0x67, 0x04, 0x94, 0x70, 0x1d, 0x40, 0x95, 0x94, 0xae, 0x86, 0x96,
-	0xd6, 0x45, 0x64, 0x1b, 0x81, 0x07, 0x7f, 0xb7, 0xa0, 0x7b, 0x40, 0xfd, 0x33, 0x4a, 0x43, 0x1c,
-	0xff, 0x09, 0x19, 0xe5, 0xb9, 0x55, 0xfe, 0x21, 0x20, 0xb7, 0x67, 0x93, 0x68, 0xee, 0x1f, 0xc8,
-	0xe0, 0xce, 0xbb, 0x8e, 0x65, 0xcf, 0x74, 0x85, 0xec, 0x43, 0xc7, 0xf8, 0xe2, 0x26, 0x6b, 0x86,
-	0x62, 0xe5, 0x47, 0x62, 0xb0, 0x7e, 0x81, 0xd4, 0xb4, 0x66, 0x0c, 0x3a, 0xd3, 0x5a, 0x75, 0xb4,
-	0x9a, 0xd6, 0xe6, 0x4d, 0x47, 0x65, 0xcd, 0x18, 0x62, 0xa6, 0xb5, 0xea, 0xd8, 0x34, 0xad, 0xcd,
-	0x9b, 0x7c, 0xca, 0x9a, 0x31, 0x69, 0x4c, 0x6b, 0xd5, 0x89, 0x68, 0x5a, 0x9b, 0x37, 0x9e, 0xae,
-	0x90, 0xef, 0x61, 0xa9, 0x32, 0x03, 0x88, 0x33, 0xd5, 0xba, 0x68, 0x78, 0x0d, 0x36, 0x2e, 0x3d,
-	0x53, 0xd8, 0x7f, 0x09, 0x5d, 0xb3, 0x37, 0x13, 0x83, 0xd0, 0x9c, 0xe9, 0x32, 0xb8, 0x79, 0x91,
-	0xd8, 0x34, 0x68, 0xb6, 0x1d, 0xd3, 0xe0, 0x9c, 0xc6, 0x6b, 0x1a, 0x9c, 0xd7, 0xad, 0x9c, 0x2b,
-	0xe4, 0x3b, 0x58, 0x9c, 0x2d, 0x7f, 0x72, 0x6b, 0x36, 0x6c, 0x95, 0xae, 0x32, 0x70, 0x2e, 0x3b,
-	0x52, 0x18, 0x7f, 0x06, 0x30, 0xad, 0x6a, 0xb2, 0x3a, 0xd5, 0xa9, 0x74, 0x95, 0xc1, 0xda, 0x7c,
-	0x61, 0x6e, 0xea, 0xe9, 0x4d, 0x58, 0x14, 0xba, 0xb4, 0x8e, 0xc4, 0x56, 0xc0, 0x19, 0x8d, 0xe4,
-	0x53, 0x50, 0x55, 0xf6, 0x0a, 0x7f, 0xeb, 0x0f, 0x5b, 0xea, 0xef, 0xfe, 0xf3, 0x7f, 0x03, 0x00,
-	0x00, 0xff, 0xff, 0x34, 0x05, 0x1f, 0x0d, 0xec, 0x0f, 0x00, 0x00,
+	// 1506 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xb4, 0x58, 0x4b, 0x6f, 0xdb, 0xc6,
+	0x16, 0x0e, 0xf5, 0xe6, 0x91, 0x94, 0xd8, 0x63, 0xdf, 0x1b, 0x46, 0xb6, 0x73, 0x15, 0xfa, 0x26,
+	0x70, 0x70, 0x03, 0xdf, 0x20, 0xed, 0x22, 0x69, 0x50, 0xa0, 0x89, 0x1f, 0x80, 0x51, 0xe7, 0x01,
+	0x3a, 0x29, 0x5a, 0x14, 0x28, 0x41, 0x93, 0x23, 0x79, 0x60, 0x8a, 0xa3, 0x92, 0x43, 0x3b, 0xe9,
+	0x4f, 0xe8, 0xa6, 0xfb, 0x02, 0x5d, 0xf7, 0x4f, 0x14, 0xdd, 0xf4, 0x57, 0xf4, 0x4f, 0x74, 0xd9,
+	0x75, 0x31, 0x67, 0x48, 0x6a, 0x28, 0xca, 0x4e, 0x8a, 0x22, 0x3b, 0xce, 0x79, 0x7c, 0xe7, 0x31,
+	0xe7, 0x31, 0x12, 0x74, 0x47, 0x2c, 0xa4, 0xf1, 0xf6, 0x34, 0xe6, 0x82, 0x93, 0x0e, 0x1e, 0xdc,
+	0xe9, 0xb1, 0xfd, 0x02, 0xd6, 0x0e, 0x39, 0x3f, 0x4d, 0xa7, 0xbb, 0x2c, 0xa6, 0xbe, 0xe0, 0xf1,
+	0xdb, 0xbd, 0x48, 0xc4, 0x6f, 0x1d, 0xfa, 0x6d, 0x4a, 0x13, 0x41, 0xd6, 0xc1, 0x0c, 0x72, 0x86,
+	0x65, 0x0c, 0x8d, 0x2d, 0xd3, 0x99, 0x11, 0x08, 0x81, 0x46, 0xe4, 0x4d, 0xa8, 0x55, 0x43, 0x06,
+	0x7e, 0xdb, 0x7b, 0xb0, 0xbe, 0x18, 0x30, 0x99, 0xf2, 0x28, 0xa1, 0xe4, 0x36, 0x34, 0xa9, 0x24,
+	0x20, 0x5a, 0xf7, 0xc1, 0xb5, 0xed, 0xdc, 0x95, 0x6d, 0x25, 0xa7, 0xb8, 0xf6, 0xaf, 0x06, 0x90,
+	0x43, 0x96, 0x08, 0x49, 0x64, 0x34, 0x79, 0x3f, 0x7f, 0xfe, 0x0d, 0xad, 0x69, 0x4c, 0x47, 0xec,
+	0x4d, 0xe6, 0x51, 0x76, 0x22, 0xf7, 0x60, 0x39, 0x11, 0x5e, 0x2c, 0xf6, 0x63, 0x3e, 0xd9, 0x67,
+	0x21, 0x7d, 0x2e, 0x9d, 0xae, 0xa3, 0x48, 0x95, 0x41, 0xb6, 0x81, 0xb0, 0xc8, 0x0f, 0xd3, 0x84,
+	0x9d, 0xd1, 0xa3, 0x9c, 0x6b, 0x35, 0x86, 0xc6, 0x56, 0xc7, 0x59, 0xc0, 0x21, 0xab, 0xd0, 0x0c,
+	0xd9, 0x84, 0x09, 0xab, 0x39, 0x34, 0xb6, 0xfa, 0x8e, 0x3a, 0xd8, 0x9f, 0xc1, 0x4a, 0xc9, 0xff,
+	0x2c, 0xfc, 0xbb, 0xd0, 0xa6, 0x8a, 0x64, 0x19, 0xc3, 0xfa, 0xa2, 0x04, 0xe4, 0x7c, 0xfb, 0xa7,
+	0x1a, 0x34, 0x91, 0x54, 0xe4, 0xd9, 0x98, 0xe5, 0x99, 0xdc, 0x82, 0x1e, 0x4b, 0xdc, 0x59, 0x32,
+	0x6a, 0xe8, 0x5f, 0x97, 0x25, 0x45, 0xde, 0xc9, 0xff, 0xa0, 0xe5, 0x9f, 0xa4, 0xd1, 0x69, 0x62,
+	0xd5, 0xd1, 0xd4, 0xca, 0xcc, 0x94, 0x0c, 0x76, 0x47, 0xf2, 0x9c, 0x4c, 0x84, 0x3c, 0x04, 0xf0,
+	0x84, 0x88, 0xd9, 0x71, 0x2a, 0x68, 0x82, 0xd1, 0x76, 0x1f, 0x58, 0x9a, 0x42, 0x9a, 0xd0, 0x27,
+	0x05, 0xdf, 0xd1, 0x64, 0xc9, 0x23, 0xe8, 0xd0, 0x37, 0x82, 0x46, 0x01, 0x0d, 0xac, 0x26, 0x1a,
+	0xda, 0x98, 0x8b, 0x69, 0x7b, 0x2f, 0xe3, 0xab, 0x08, 0x0b, 0xf1, 0xc1, 0x63, 0xe8, 0x97, 0x58,
+	0x64, 0x09, 0xea, 0xa7, 0x34, 0xbf, 0x59, 0xf9, 0x29, 0xb3, 0x7b, 0xe6, 0x85, 0xa9, 0x2a, 0xb2,
+	0x9e, 0xa3, 0x0e, 0x9f, 0xd4, 0x1e, 0x1a, 0xf6, 0x2e, 0x98, 0xfb, 0x69, 0x18, 0x16, 0x8a, 0x01,
+	0x8b, 0x73, 0xc5, 0x80, 0xc5, 0xb3, 0x42, 0xab, 0x5d, 0x5a, 0x68, 0xbf, 0x18, 0xb0, 0xbc, 0x77,
+	0x46, 0x23, 0xf1, 0x9c, 0x0b, 0x36, 0x62, 0xbe, 0x27, 0x18, 0x8f, 0xc8, 0x3d, 0x30, 0x79, 0x18,
+	0xb8, 0x97, 0x56, 0x6a, 0x87, 0x87, 0x99, 0xd7, 0xf7, 0xc0, 0x8c, 0xe8, 0xb9, 0x7b, 0xa9, 0xb9,
+	0x4e, 0x44, 0xcf, 0x95, 0xf4, 0x26, 0xf4, 0x03, 0x1a, 0x52, 0x41, 0xdd, 0xe2, 0x76, 0xe4, 0xd5,
+	0xf5, 0x14, 0x71, 0x47, 0x5d, 0xc7, 0x1d, 0xb8, 0x26, 0x21, 0xa7, 0x5e, 0x4c, 0x23, 0xe1, 0x4e,
+	0x3d, 0x71, 0x82, 0x77, 0x62, 0x3a, 0xfd, 0x88, 0x9e, 0xbf, 0x44, 0xea, 0x4b, 0x4f, 0x9c, 0xd8,
+	0x7f, 0x1a, 0x60, 0x16, 0x97, 0x49, 0xae, 0x43, 0x5b, 0x9a, 0x75, 0x59, 0x90, 0x65, 0xa2, 0x25,
+	0x8f, 0x07, 0x81, 0xec, 0x0c, 0x3e, 0x1a, 0x25, 0x54, 0xa0, 0x7b, 0x75, 0x27, 0x3b, 0xc9, 0xca,
+	0x4a, 0xd8, 0x77, 0xaa, 0x19, 0x1a, 0x0e, 0x7e, 0xcb, 0x8c, 0x4f, 0x04, 0x9b, 0x50, 0x34, 0x58,
+	0x77, 0xd4, 0x81, 0xac, 0x40, 0x93, 0xba, 0xc2, 0x1b, 0x63, 0x95, 0x9b, 0x4e, 0x83, 0xbe, 0xf2,
+	0xc6, 0xe4, 0xbf, 0x70, 0x35, 0xe1, 0x69, 0xec, 0x53, 0x37, 0x37, 0xdb, 0x42, 0x6e, 0x4f, 0x51,
+	0xf7, 0x95, 0x71, 0x1b, 0xea, 0x23, 0x16, 0x58, 0x6d, 0x4c, 0xcc, 0x52, 0xb9, 0x08, 0x0f, 0x02,
+	0x47, 0x32, 0xc9, 0xff, 0x01, 0x0a, 0xa4, 0xc0, 0xea, 0x5c, 0x20, 0x6a, 0xe6, 0xb8, 0x81, 0xfd,
+	0x25, 0xb4, 0x32, 0xf8, 0x35, 0x30, 0xcf, 0x78, 0x98, 0x4e, 0x8a, 0xb0, 0xfb, 0x4e, 0x47, 0x11,
+	0x0e, 0x02, 0x72, 0x03, 0x70, 0xd6, 0xb9, 0xb2, 0xaa, 0x6a, 0x18, 0x24, 0x66, 0xe8, 0x73, 0x8a,
+	0xd3, 0xc2, 0xe7, 0xfc, 0x94, 0xa9, 0xe8, 0xdb, 0x4e, 0x76, 0xb2, 0xff, 0xa8, 0xc1, 0xd5, 0x72,
+	0xb9, 0x4b, 0x13, 0x88, 0x82, 0xb9, 0x32, 0x10, 0x06, 0x61, 0x8f, 0x4a, 0xf9, 0xaa, 0xe9, 0xf9,
+	0xca, 0x55, 0x26, 0x3c, 0x50, 0x06, 0xfa, 0x4a, 0xe5, 0x19, 0x0f, 0xa8, 0xac, 0xd6, 0x94, 0x05,
+	0x98, 0xe0, 0xbe, 0x23, 0x3f, 0x25, 0x65, 0xcc, 0x82, 0x6c, 0x84, 0xc8, 0x4f, 0x74, 0x2f, 0x46,
+	0xdc, 0x96, 0xba, 0x32, 0x75, 0x92, 0x57, 0x36, 0x91, 0xd4, 0xb6, 0xba, 0x07, 0xf9, 0x4d, 0x86,
+	0xd0, 0x8d, 0xe9, 0x34, 0xcc, 0xaa, 0x17, 0xd3, 0x67, 0x3a, 0x3a, 0x89, 0xdc, 0x04, 0xf0, 0x79,
+	0x18, 0x52, 0x1f, 0x05, 0x4c, 0x14, 0xd0, 0x28, 0xb2, 0x72, 0x84, 0x08, 0xdd, 0x84, 0xfa, 0x16,
+	0x0c, 0x8d, 0xad, 0xa6, 0xd3, 0x12, 0x22, 0x3c, 0xa2, 0xbe, 0x8c, 0x23, 0x4d, 0x68, 0xec, 0xe2,
+	0x00, 0xea, 0xa2, 0x5e, 0x47, 0x12, 0x70, 0x54, 0x6e, 0x00, 0x8c, 0x63, 0x9e, 0x4e, 0x15, 0xb7,
+	0x37, 0xac, 0xcb, 0x79, 0x8c, 0x14, 0x64, 0xdf, 0x86, 0xab, 0xc9, 0xdb, 0x49, 0xc8, 0xa2, 0x53,
+	0x57, 0x78, 0xf1, 0x98, 0x0a, 0xab, 0xaf, 0x6a, 0x38, 0xa3, 0xbe, 0x42, 0xa2, 0xfd, 0x15, 0x90,
+	0x9d, 0x98, 0x7a, 0x82, 0xfe, 0x8d, 0xd5, 0xf3, 0x9e, 0xdd, 0xfd, 0x2f, 0x58, 0x29, 0x41, 0xab,
+	0x29, 0x2c, 0x2d, 0xbe, 0x9e, 0x06, 0x1f, 0xca, 0x62, 0x09, 0x3a, 0xb3, 0xf8, 0x83, 0x01, 0x64,
+	0x17, 0x1b, 0xfc, 0x9f, 0xed, 0x57, 0xd9, 0x72, 0x72, 0xee, 0xab, 0x01, 0x12, 0x78, 0xc2, 0xcb,
+	0x36, 0x53, 0x8f, 0x25, 0x0a, 0x7f, 0xd7, 0x13, 0x5e, 0xb6, 0x1d, 0x62, 0xea, 0xa7, 0xb1, 0x5c,
+	0x56, 0x58, 0x57, 0xb8, 0x1d, 0x9c, 0x9c, 0x24, 0x1d, 0x2d, 0x39, 0x94, 0x39, 0xfa, 0xa3, 0x01,
+	0xd6, 0x13, 0xc1, 0x27, 0xcc, 0x77, 0xa8, 0x34, 0x58, 0x72, 0x77, 0x13, 0xfa, 0x72, 0x2c, 0xce,
+	0xbb, 0xdc, 0xe3, 0x61, 0x30, 0x5b, 0x3b, 0x37, 0x40, 0x4e, 0x46, 0x57, 0xf3, 0xbc, 0xcd, 0xc3,
+	0x00, 0x0b, 0x62, 0x13, 0xe4, 0xf8, 0xd2, 0xf4, 0xd5, 0x12, 0xee, 0x45, 0xf4, 0xbc, 0xa4, 0x2f,
+	0x85, 0x50, 0x5f, 0xcd, 0xbc, 0x76, 0x44, 0xcf, 0xa5, 0xbe, 0xbd, 0x06, 0x37, 0x16, 0xf8, 0x96,
+	0x79, 0xfe, 0xb3, 0x01, 0x2b, 0x4f, 0x92, 0x84, 0x8d, 0xa3, 0x2f, 0xb0, 0xfb, 0x73, 0xa7, 0x57,
+	0xa1, 0xe9, 0xf3, 0x34, 0x12, 0xe8, 0x6c, 0xd3, 0x51, 0x87, 0xb9, 0x86, 0xa8, 0x55, 0x1a, 0x62,
+	0xae, 0xa5, 0xea, 0xd5, 0x96, 0xd2, 0x5a, 0xa6, 0x51, 0x6a, 0x99, 0xff, 0x40, 0x57, 0x5e, 0x8c,
+	0xeb, 0xd3, 0x48, 0xd0, 0x38, 0x1b, 0x98, 0x20, 0x49, 0x3b, 0x48, 0xb1, 0xbf, 0x37, 0x60, 0xb5,
+	0xec, 0x69, 0xf6, 0x3a, 0xb8, 0x70, 0x7e, 0xcb, 0x81, 0x11, 0x87, 0x99, 0x9b, 0xf2, 0x53, 0xb6,
+	0xde, 0x34, 0x3d, 0x0e, 0x99, 0xef, 0x4a, 0x86, 0x72, 0xcf, 0x54, 0x94, 0xd7, 0x71, 0x38, 0x0b,
+	0xba, 0xa1, 0x07, 0x4d, 0xa0, 0xe1, 0xa5, 0xe2, 0x24, 0x9f, 0xe1, 0xf2, 0xdb, 0xfe, 0x18, 0x56,
+	0xd4, 0x83, 0xad, 0x9c, 0xb5, 0x0d, 0x80, 0x62, 0xaa, 0xaa, 0xb7, 0x8a, 0xe9, 0x98, 0xf9, 0x58,
+	0x4d, 0xec, 0x4f, 0xc1, 0x3c, 0xe4, 0x2a, 0x11, 0x09, 0xb9, 0x0f, 0x66, 0x98, 0x1f, 0xb2, 0x67,
+	0x0d, 0x99, 0xb5, 0x47, 0x2e, 0xe7, 0xcc, 0x84, 0xec, 0xc7, 0xd0, 0xc9, 0xc9, 0x79, 0x6c, 0xc6,
+	0x45, 0xb1, 0xd5, 0xe6, 0x62, 0xb3, 0x7f, 0x33, 0x60, 0xb5, 0xec, 0x72, 0x96, 0xbe, 0xd7, 0xd0,
+	0x2f, 0x4c, 0xb8, 0x13, 0x6f, 0x9a, 0xf9, 0x72, 0x5f, 0xf7, 0xa5, 0xaa, 0x56, 0x38, 0x98, 0x3c,
+	0xf3, 0xa6, 0xaa, 0xa4, 0x7a, 0xa1, 0x46, 0x1a, 0xbc, 0x82, 0xe5, 0x8a, 0xc8, 0x82, 0x97, 0xca,
+	0x5d, 0xfd, 0xa5, 0x52, 0x7a, 0x6d, 0x15, 0xda, 0xfa, 0xf3, 0xe5, 0x11, 0x5c, 0x57, 0xfd, 0xb7,
+	0x53, 0x14, 0x5d, 0x9e, 0xfb, 0x72, 0x6d, 0x1a, 0xf3, 0xb5, 0x69, 0x0f, 0xc0, 0xaa, 0xaa, 0x66,
+	0x5d, 0x30, 0x86, 0xe5, 0x23, 0xe1, 0x09, 0x96, 0x08, 0xe6, 0x17, 0xcf, 0xe6, 0xb9, 0x62, 0x36,
+	0xde, 0xb5, 0x1f, 0xaa, 0xed, 0xb0, 0x04, 0x75, 0x21, 0xf2, 0x3a, 0x93, 0x9f, 0xf2, 0x16, 0x88,
+	0x6e, 0x29, 0xbb, 0x83, 0x0f, 0x60, 0x4a, 0xd6, 0x83, 0xe0, 0xc2, 0x0b, 0xd5, 0xfe, 0x6d, 0xe0,
+	0xfe, 0x35, 0x91, 0x82, 0x0b, 0x58, 0xad, 0xa8, 0x40, 0x71, 0x9b, 0x6a, 0x3b, 0x4b, 0x02, 0x32,
+	0x37, 0x00, 0xb0, 0xa5, 0x54, 0x37, 0xb4, 0x94, 0xae, 0xa4, 0xec, 0x48, 0xc2, 0x83, 0xdf, 0x5b,
+	0xd0, 0x3b, 0xa2, 0xde, 0x39, 0xa5, 0x81, 0x7c, 0x4e, 0xc4, 0x64, 0x9c, 0xd7, 0x56, 0xf9, 0xf7,
+	0x0b, 0xb9, 0x3d, 0x5f, 0x44, 0x0b, 0x7f, 0x30, 0x0d, 0xee, 0xbc, 0x4b, 0x2c, 0xbb, 0xa6, 0x2b,
+	0xe4, 0x10, 0xba, 0xda, 0x0f, 0x04, 0xb2, 0xae, 0x29, 0x56, 0x7e, 0xf7, 0x0c, 0x36, 0x2e, 0xe0,
+	0xea, 0x68, 0xda, 0xa2, 0xd3, 0xd1, 0xaa, 0xab, 0x55, 0x47, 0x5b, 0xb4, 0x1d, 0x11, 0x4d, 0x5b,
+	0x62, 0x3a, 0x5a, 0x75, 0x6d, 0xea, 0x68, 0x8b, 0x36, 0x1f, 0xa2, 0x69, 0x9b, 0x46, 0x47, 0xab,
+	0x6e, 0x44, 0x1d, 0x6d, 0xd1, 0x7a, 0xba, 0x42, 0xbe, 0x81, 0xe5, 0xca, 0x0e, 0x20, 0xf6, 0x4c,
+	0xeb, 0xa2, 0xe5, 0x35, 0xd8, 0xbc, 0x54, 0xa6, 0xc0, 0x7f, 0x01, 0x3d, 0x7d, 0x36, 0x13, 0xcd,
+	0xa1, 0x05, 0xdb, 0x65, 0x70, 0xf3, 0x22, 0xb6, 0x0e, 0xa8, 0x8f, 0x1d, 0x1d, 0x70, 0xc1, 0xe0,
+	0xd5, 0x01, 0x17, 0x4d, 0x2b, 0xfb, 0x0a, 0xf9, 0x1a, 0x96, 0xe6, 0xdb, 0x9f, 0xdc, 0x9a, 0x4f,
+	0x5b, 0x65, 0xaa, 0x0c, 0xec, 0xcb, 0x44, 0x0a, 0xf0, 0x03, 0x80, 0x59, 0x57, 0x93, 0xb5, 0x99,
+	0x4e, 0x65, 0xaa, 0x0c, 0xd6, 0x17, 0x33, 0x73, 0xa8, 0xa7, 0x37, 0x61, 0x29, 0x51, 0xad, 0x35,
+	0x4a, 0xb6, 0xfd, 0x90, 0xd1, 0x48, 0x3c, 0x05, 0xec, 0xb2, 0x97, 0x31, 0x17, 0xfc, 0xb8, 0x85,
+	0x7f, 0x46, 0x7c, 0xf4, 0x57, 0x00, 0x00, 0x00, 0xff, 0xff, 0x1c, 0xce, 0x1a, 0x06, 0x9b, 0x10,
+	0x00, 0x00,
 }
diff --git a/weed/pb/filer_pb/filer_pb_helper.go b/weed/pb/filer_pb/filer_pb_helper.go
new file mode 100644
index 000000000..1017e6910
--- /dev/null
+++ b/weed/pb/filer_pb/filer_pb_helper.go
@@ -0,0 +1,58 @@
+package filer_pb
+
+import (
+	"github.com/chrislusf/seaweedfs/weed/storage/needle"
+)
+
+func toFileId(fileIdStr string) (*FileId, error) {
+	t, err := needle.ParseFileIdFromString(fileIdStr)
+	if err != nil {
+		return nil, err
+	}
+	return &FileId{
+		VolumeId: uint32(t.VolumeId),
+		Cookie:   uint32(t.Cookie),
+		FileKey:  uint64(t.Key),
+	}, nil
+
+}
+
+func (fid *FileId) toFileId() string {
+	return needle.NewFileId(needle.VolumeId(fid.VolumeId), fid.FileKey, fid.Cookie).String()
+}
+
+func BeforeEntrySerialization(chunks []*FileChunk) {
+
+	for _, chunk := range chunks {
+
+		if chunk.FileId != "" {
+			if fid, err := toFileId(chunk.FileId); err == nil {
+				chunk.Fid = fid
+				chunk.FileId = ""
+			}
+		}
+
+		if chunk.SourceFileId != "" {
+			if fid, err := toFileId(chunk.SourceFileId); err == nil {
+				chunk.SourceFid = fid
+				chunk.SourceFileId = ""
+			}
+		}
+
+	}
+}
+
+func AfterEntryDeserialization(chunks []*FileChunk) {
+
+	for _, chunk := range chunks {
+
+		if chunk.Fid != nil && chunk.FileId == "" {
+			chunk.FileId = chunk.Fid.toFileId()
+		}
+
+		if chunk.SourceFid != nil && chunk.SourceFileId == "" {
+			chunk.SourceFileId = chunk.SourceFid.toFileId()
+		}
+
+	}
+}
diff --git a/weed/pb/filer_pb/filer_pb_helper_test.go b/weed/pb/filer_pb/filer_pb_helper_test.go
new file mode 100644
index 000000000..6f08a1159
--- /dev/null
+++ b/weed/pb/filer_pb/filer_pb_helper_test.go
@@ -0,0 +1,18 @@
+package filer_pb
+
+import (
+	"testing"
+
+	"github.com/gogo/protobuf/proto"
+)
+
+func TestFileIdSize(t *testing.T) {
+	fileIdStr := "11745,0293434534cbb9892b"
+
+	fid, _ := toFileId(fileIdStr)
+	bytes, _ := proto.Marshal(fid)
+
+	println(len(fileIdStr))
+	println(len(bytes))
+}
+

From bf9d4905382eec3f4a2095947fc9bec997fd070e Mon Sep 17 00:00:00 2001
From: Chris Lu <chris.lu@gmail.com>
Date: Fri, 17 May 2019 02:28:20 -0700
Subject: [PATCH 05/10] fix for tests

---
 weed/filer2/filerstore.go                | 3 +++
 weed/pb/filer_pb/filer_pb_helper_test.go | 2 +-
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/weed/filer2/filerstore.go b/weed/filer2/filerstore.go
index 7755ee1a3..3db50bf17 100644
--- a/weed/filer2/filerstore.go
+++ b/weed/filer2/filerstore.go
@@ -56,6 +56,9 @@ func (fsw *FilerStoreWrapper) UpdateEntry(ctx context.Context, entry *Entry) err
 
 func (fsw *FilerStoreWrapper) FindEntry(ctx context.Context, fp FullPath) (entry *Entry, err error) {
 	entry, err = fsw.actualStore.FindEntry(ctx, fp)
+	if err != nil {
+		return nil, err
+	}
 	filer_pb.AfterEntryDeserialization(entry.Chunks)
 	return
 }
diff --git a/weed/pb/filer_pb/filer_pb_helper_test.go b/weed/pb/filer_pb/filer_pb_helper_test.go
index 6f08a1159..64ed72ed7 100644
--- a/weed/pb/filer_pb/filer_pb_helper_test.go
+++ b/weed/pb/filer_pb/filer_pb_helper_test.go
@@ -3,7 +3,7 @@ package filer_pb
 import (
 	"testing"
 
-	"github.com/gogo/protobuf/proto"
+	"github.com/golang/protobuf/proto"
 )
 
 func TestFileIdSize(t *testing.T) {

From 7e0c3bb73219e625016aed26dda5047478c58753 Mon Sep 17 00:00:00 2001
From: Chris Lu <chris.lu@gmail.com>
Date: Fri, 17 May 2019 09:11:08 -0700
Subject: [PATCH 06/10] filer: adjust recommended mysql meta data type to
 LONGBLOB

fix https://github.com/chrislusf/seaweedfs/issues/966
---
 weed/command/scaffold.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/weed/command/scaffold.go b/weed/command/scaffold.go
index 1f1d09ff0..e56a1b5ba 100644
--- a/weed/command/scaffold.go
+++ b/weed/command/scaffold.go
@@ -75,7 +75,7 @@ dir = "."					# directory to store level db files
 #   dirhash     BIGINT         COMMENT 'first 64 bits of MD5 hash value of directory field',
 #   name        VARCHAR(1000)  COMMENT 'directory or file name',
 #   directory   TEXT           COMMENT 'full path to parent directory',
-#   meta        BLOB,
+#   meta        LONGBLOB,
 #   PRIMARY KEY (dirhash, name)
 # ) DEFAULT CHARSET=utf8;
 

From 0fe286a6cac0dc2d7cb960645a4235c25516ea91 Mon Sep 17 00:00:00 2001
From: Chris Lu <chris.lu@gmail.com>
Date: Fri, 17 May 2019 17:33:49 -0700
Subject: [PATCH 07/10] filer: optimization for leveldb, add leveldb2

large filer db may see high CPU and disk usage due to background compaction
---
 .../compact_leveldb/compact_leveldb.go        |  35 ++++
 weed/command/scaffold.go                      |   6 +
 weed/filer2/leveldb/leveldb_store.go          |   9 +-
 weed/filer2/leveldb2/leveldb2_store.go        | 190 ++++++++++++++++++
 weed/filer2/leveldb2/leveldb2_store_test.go   |  88 ++++++++
 5 files changed, 327 insertions(+), 1 deletion(-)
 create mode 100644 unmaintained/compact_leveldb/compact_leveldb.go
 create mode 100644 weed/filer2/leveldb2/leveldb2_store.go
 create mode 100644 weed/filer2/leveldb2/leveldb2_store_test.go

diff --git a/unmaintained/compact_leveldb/compact_leveldb.go b/unmaintained/compact_leveldb/compact_leveldb.go
new file mode 100644
index 000000000..317356c3f
--- /dev/null
+++ b/unmaintained/compact_leveldb/compact_leveldb.go
@@ -0,0 +1,35 @@
+package main
+
+import (
+	"flag"
+	"log"
+
+	"github.com/syndtr/goleveldb/leveldb"
+	"github.com/syndtr/goleveldb/leveldb/opt"
+	"github.com/syndtr/goleveldb/leveldb/util"
+)
+
+var (
+	dir = flag.String("dir", ".", "data directory to store leveldb files")
+)
+
+func main() {
+
+	flag.Parse()
+
+	opts := &opt.Options{
+		BlockCacheCapacity:            32 * 1024 * 1024, // default value is 8MiB
+		WriteBuffer:                   16 * 1024 * 1024, // default value is 4MiB
+		CompactionTableSizeMultiplier: 10,
+		OpenFilesCacheCapacity:        -1,
+	}
+
+	db, err := leveldb.OpenFile(*dir, opts)
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer db.Close()
+	if err := db.CompactRange(util.Range{}); err != nil {
+		log.Fatal(err)
+	}
+}
diff --git a/weed/command/scaffold.go b/weed/command/scaffold.go
index e56a1b5ba..a0908912c 100644
--- a/weed/command/scaffold.go
+++ b/weed/command/scaffold.go
@@ -63,6 +63,12 @@ enabled = false
 
 [leveldb]
 # local on disk, mostly for simple single-machine setup, fairly scalable
+enabled = false
+dir = "."					# directory to store level db files
+
+[leveldb2]
+# local on disk, mostly for simple single-machine setup, fairly scalable
+# faster than previous leveldb, recommended.
 enabled = true
 dir = "."					# directory to store level db files
 
diff --git a/weed/filer2/leveldb/leveldb_store.go b/weed/filer2/leveldb/leveldb_store.go
index 5b3a63959..a55c58153 100644
--- a/weed/filer2/leveldb/leveldb_store.go
+++ b/weed/filer2/leveldb/leveldb_store.go
@@ -9,6 +9,7 @@ import (
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	weed_util "github.com/chrislusf/seaweedfs/weed/util"
 	"github.com/syndtr/goleveldb/leveldb"
+	"github.com/syndtr/goleveldb/leveldb/opt"
 	leveldb_util "github.com/syndtr/goleveldb/leveldb/util"
 )
 
@@ -39,7 +40,13 @@ func (store *LevelDBStore) initialize(dir string) (err error) {
 		return fmt.Errorf("Check Level Folder %s Writable: %s", dir, err)
 	}
 
-	if store.db, err = leveldb.OpenFile(dir, nil); err != nil {
+	opts := &opt.Options{
+		BlockCacheCapacity:            32 * 1024 * 1024, // default value is 8MiB
+		WriteBuffer:                   16 * 1024 * 1024, // default value is 4MiB
+		CompactionTableSizeMultiplier: 4,
+	}
+
+	if store.db, err = leveldb.OpenFile(dir, opts); err != nil {
 		glog.Infof("filer store open dir %s: %v", dir, err)
 		return
 	}
diff --git a/weed/filer2/leveldb2/leveldb2_store.go b/weed/filer2/leveldb2/leveldb2_store.go
new file mode 100644
index 000000000..c0e2cd877
--- /dev/null
+++ b/weed/filer2/leveldb2/leveldb2_store.go
@@ -0,0 +1,190 @@
+package leveldb
+
+import (
+	"bytes"
+	"context"
+	"crypto/md5"
+	"fmt"
+	"io"
+
+	"github.com/chrislusf/seaweedfs/weed/filer2"
+	"github.com/chrislusf/seaweedfs/weed/glog"
+	weed_util "github.com/chrislusf/seaweedfs/weed/util"
+	"github.com/syndtr/goleveldb/leveldb"
+	"github.com/syndtr/goleveldb/leveldb/opt"
+	leveldb_util "github.com/syndtr/goleveldb/leveldb/util"
+)
+
+func init() {
+	filer2.Stores = append(filer2.Stores, &LevelDB2Store{})
+}
+
+// known theoretically 128 bit MD5 collision of 2 directories.
+// (but really? please show some real examples)
+type LevelDB2Store struct {
+	db *leveldb.DB
+}
+
+func (store *LevelDB2Store) GetName() string {
+	return "leveldb2"
+}
+
+func (store *LevelDB2Store) Initialize(configuration weed_util.Configuration) (err error) {
+	dir := configuration.GetString("dir")
+	return store.initialize(dir)
+}
+
+func (store *LevelDB2Store) initialize(dir string) (err error) {
+	glog.Infof("filer store leveldb2 dir: %s", dir)
+	if err := weed_util.TestFolderWritable(dir); err != nil {
+		return fmt.Errorf("Check Level Folder %s Writable: %s", dir, err)
+	}
+
+	opts := &opt.Options{
+		BlockCacheCapacity:            32 * 1024 * 1024, // default value is 8MiB
+		WriteBuffer:                   16 * 1024 * 1024, // default value is 4MiB
+		CompactionTableSizeMultiplier: 10,
+	}
+
+	if store.db, err = leveldb.OpenFile(dir, opts); err != nil {
+		glog.Infof("filer store open dir %s: %v", dir, err)
+		return
+	}
+	return
+}
+
+func (store *LevelDB2Store) BeginTransaction(ctx context.Context) (context.Context, error) {
+	return ctx, nil
+}
+func (store *LevelDB2Store) CommitTransaction(ctx context.Context) error {
+	return nil
+}
+func (store *LevelDB2Store) RollbackTransaction(ctx context.Context) error {
+	return nil
+}
+
+func (store *LevelDB2Store) InsertEntry(ctx context.Context, entry *filer2.Entry) (err error) {
+	key := genKey(entry.DirAndName())
+
+	value, err := entry.EncodeAttributesAndChunks()
+	if err != nil {
+		return fmt.Errorf("encoding %s %+v: %v", entry.FullPath, entry.Attr, err)
+	}
+
+	err = store.db.Put(key, value, nil)
+
+	if err != nil {
+		return fmt.Errorf("persisting %s : %v", entry.FullPath, err)
+	}
+
+	// println("saved", entry.FullPath, "chunks", len(entry.Chunks))
+
+	return nil
+}
+
+func (store *LevelDB2Store) UpdateEntry(ctx context.Context, entry *filer2.Entry) (err error) {
+
+	return store.InsertEntry(ctx, entry)
+}
+
+func (store *LevelDB2Store) FindEntry(ctx context.Context, fullpath filer2.FullPath) (entry *filer2.Entry, err error) {
+	key := genKey(fullpath.DirAndName())
+
+	data, err := store.db.Get(key, nil)
+
+	if err == leveldb.ErrNotFound {
+		return nil, filer2.ErrNotFound
+	}
+	if err != nil {
+		return nil, fmt.Errorf("get %s : %v", entry.FullPath, err)
+	}
+
+	entry = &filer2.Entry{
+		FullPath: fullpath,
+	}
+	err = entry.DecodeAttributesAndChunks(data)
+	if err != nil {
+		return entry, fmt.Errorf("decode %s : %v", entry.FullPath, err)
+	}
+
+	// println("read", entry.FullPath, "chunks", len(entry.Chunks), "data", len(data), string(data))
+
+	return entry, nil
+}
+
+func (store *LevelDB2Store) DeleteEntry(ctx context.Context, fullpath filer2.FullPath) (err error) {
+	key := genKey(fullpath.DirAndName())
+
+	err = store.db.Delete(key, nil)
+	if err != nil {
+		return fmt.Errorf("delete %s : %v", fullpath, err)
+	}
+
+	return nil
+}
+
+func (store *LevelDB2Store) ListDirectoryEntries(ctx context.Context, fullpath filer2.FullPath, startFileName string, inclusive bool,
+	limit int) (entries []*filer2.Entry, err error) {
+
+	directoryPrefix := genDirectoryKeyPrefix(fullpath, "")
+
+	iter := store.db.NewIterator(&leveldb_util.Range{Start: genDirectoryKeyPrefix(fullpath, startFileName)}, nil)
+	for iter.Next() {
+		key := iter.Key()
+		if !bytes.HasPrefix(key, directoryPrefix) {
+			break
+		}
+		fileName := getNameFromKey(key)
+		if fileName == "" {
+			continue
+		}
+		if fileName == startFileName && !inclusive {
+			continue
+		}
+		limit--
+		if limit < 0 {
+			break
+		}
+		entry := &filer2.Entry{
+			FullPath: filer2.NewFullPath(string(fullpath), fileName),
+		}
+		if decodeErr := entry.DecodeAttributesAndChunks(iter.Value()); decodeErr != nil {
+			err = decodeErr
+			glog.V(0).Infof("list %s : %v", entry.FullPath, err)
+			break
+		}
+		entries = append(entries, entry)
+	}
+	iter.Release()
+
+	return entries, err
+}
+
+func genKey(dirPath, fileName string) (key []byte) {
+	key = hashToBytes(dirPath)
+	key = append(key, []byte(fileName)...)
+	return key
+}
+
+func genDirectoryKeyPrefix(fullpath filer2.FullPath, startFileName string) (keyPrefix []byte) {
+	keyPrefix = hashToBytes(string(fullpath))
+	if len(startFileName) > 0 {
+		keyPrefix = append(keyPrefix, []byte(startFileName)...)
+	}
+	return keyPrefix
+}
+
+func getNameFromKey(key []byte) string {
+
+	return string(key[8:])
+
+}
+
+func hashToBytes(dir string) []byte {
+	h := md5.New()
+	io.WriteString(h, dir)
+
+	b := h.Sum(nil)
+
+	return b
+}
diff --git a/weed/filer2/leveldb2/leveldb2_store_test.go b/weed/filer2/leveldb2/leveldb2_store_test.go
new file mode 100644
index 000000000..5d6e660de
--- /dev/null
+++ b/weed/filer2/leveldb2/leveldb2_store_test.go
@@ -0,0 +1,88 @@
+package leveldb
+
+import (
+	"context"
+	"github.com/chrislusf/seaweedfs/weed/filer2"
+	"io/ioutil"
+	"os"
+	"testing"
+)
+
+func TestCreateAndFind(t *testing.T) {
+	filer := filer2.NewFiler(nil, nil)
+	dir, _ := ioutil.TempDir("", "seaweedfs_filer_test")
+	defer os.RemoveAll(dir)
+	store := &LevelDB2Store{}
+	store.initialize(dir)
+	filer.SetStore(store)
+	filer.DisableDirectoryCache()
+
+	fullpath := filer2.FullPath("/home/chris/this/is/one/file1.jpg")
+
+	ctx := context.Background()
+
+	entry1 := &filer2.Entry{
+		FullPath: fullpath,
+		Attr: filer2.Attr{
+			Mode: 0440,
+			Uid:  1234,
+			Gid:  5678,
+		},
+	}
+
+	if err := filer.CreateEntry(ctx, entry1); err != nil {
+		t.Errorf("create entry %v: %v", entry1.FullPath, err)
+		return
+	}
+
+	entry, err := filer.FindEntry(ctx, fullpath)
+
+	if err != nil {
+		t.Errorf("find entry: %v", err)
+		return
+	}
+
+	if entry.FullPath != entry1.FullPath {
+		t.Errorf("find wrong entry: %v", entry.FullPath)
+		return
+	}
+
+	// checking one upper directory
+	entries, _ := filer.ListDirectoryEntries(ctx, filer2.FullPath("/home/chris/this/is/one"), "", false, 100)
+	if len(entries) != 1 {
+		t.Errorf("list entries count: %v", len(entries))
+		return
+	}
+
+	// checking one upper directory
+	entries, _ = filer.ListDirectoryEntries(ctx, filer2.FullPath("/"), "", false, 100)
+	if len(entries) != 1 {
+		t.Errorf("list entries count: %v", len(entries))
+		return
+	}
+
+}
+
+func TestEmptyRoot(t *testing.T) {
+	filer := filer2.NewFiler(nil, nil)
+	dir, _ := ioutil.TempDir("", "seaweedfs_filer_test2")
+	defer os.RemoveAll(dir)
+	store := &LevelDB2Store{}
+	store.initialize(dir)
+	filer.SetStore(store)
+	filer.DisableDirectoryCache()
+
+	ctx := context.Background()
+
+	// checking one upper directory
+	entries, err := filer.ListDirectoryEntries(ctx, filer2.FullPath("/"), "", false, 100)
+	if err != nil {
+		t.Errorf("list entries: %v", err)
+		return
+	}
+	if len(entries) != 0 {
+		t.Errorf("list entries count: %v", len(entries))
+		return
+	}
+
+}

From 76cf57a0e15b0b89dd3f36db641a6112fa81f44b Mon Sep 17 00:00:00 2001
From: Chris Lu <chris.lu@gmail.com>
Date: Fri, 17 May 2019 17:34:03 -0700
Subject: [PATCH 08/10] volume: optimzation for leveldb

---
 weed/storage/volume_loading.go | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/weed/storage/volume_loading.go b/weed/storage/volume_loading.go
index cfc59eb09..51fbe12e5 100644
--- a/weed/storage/volume_loading.go
+++ b/weed/storage/volume_loading.go
@@ -86,8 +86,9 @@ func (v *Volume) load(alsoLoadIndex bool, createDatIfMissing bool, needleMapKind
 		case NeedleMapLevelDb:
 			glog.V(0).Infoln("loading leveldb", fileName+".ldb")
 			opts := &opt.Options{
-				BlockCacheCapacity: 2 * 1024 * 1024, // default value is 8MiB
-				WriteBuffer:        1 * 1024 * 1024, // default value is 4MiB
+				BlockCacheCapacity:            2 * 1024 * 1024, // default value is 8MiB
+				WriteBuffer:                   1 * 1024 * 1024, // default value is 4MiB
+				CompactionTableSizeMultiplier: 10,              // default value is 1
 			}
 			if v.nm, e = NewLevelDbNeedleMap(fileName+".ldb", indexFile, opts); e != nil {
 				glog.V(0).Infof("loading leveldb %s error: %v", fileName+".ldb", e)
@@ -95,8 +96,9 @@ func (v *Volume) load(alsoLoadIndex bool, createDatIfMissing bool, needleMapKind
 		case NeedleMapLevelDbMedium:
 			glog.V(0).Infoln("loading leveldb medium", fileName+".ldb")
 			opts := &opt.Options{
-				BlockCacheCapacity: 4 * 1024 * 1024, // default value is 8MiB
-				WriteBuffer:        2 * 1024 * 1024, // default value is 4MiB
+				BlockCacheCapacity:            4 * 1024 * 1024, // default value is 8MiB
+				WriteBuffer:                   2 * 1024 * 1024, // default value is 4MiB
+				CompactionTableSizeMultiplier: 10,              // default value is 1
 			}
 			if v.nm, e = NewLevelDbNeedleMap(fileName+".ldb", indexFile, opts); e != nil {
 				glog.V(0).Infof("loading leveldb %s error: %v", fileName+".ldb", e)
@@ -104,8 +106,9 @@ func (v *Volume) load(alsoLoadIndex bool, createDatIfMissing bool, needleMapKind
 		case NeedleMapLevelDbLarge:
 			glog.V(0).Infoln("loading leveldb large", fileName+".ldb")
 			opts := &opt.Options{
-				BlockCacheCapacity: 8 * 1024 * 1024, // default value is 8MiB
-				WriteBuffer:        4 * 1024 * 1024, // default value is 4MiB
+				BlockCacheCapacity:            8 * 1024 * 1024, // default value is 8MiB
+				WriteBuffer:                   4 * 1024 * 1024, // default value is 4MiB
+				CompactionTableSizeMultiplier: 10,              // default value is 1
 			}
 			if v.nm, e = NewLevelDbNeedleMap(fileName+".ldb", indexFile, opts); e != nil {
 				glog.V(0).Infof("loading leveldb %s error: %v", fileName+".ldb", e)

From 8123a488cd73e45c9245eb2f2ca26a3d2f29cc76 Mon Sep 17 00:00:00 2001
From: Chris Lu <chris.lu@gmail.com>
Date: Fri, 17 May 2019 18:14:25 -0700
Subject: [PATCH 09/10] filer: optimize leveldb parameters

---
 weed/filer2/leveldb/leveldb_store.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/weed/filer2/leveldb/leveldb_store.go b/weed/filer2/leveldb/leveldb_store.go
index a55c58153..d00eba859 100644
--- a/weed/filer2/leveldb/leveldb_store.go
+++ b/weed/filer2/leveldb/leveldb_store.go
@@ -43,7 +43,7 @@ func (store *LevelDBStore) initialize(dir string) (err error) {
 	opts := &opt.Options{
 		BlockCacheCapacity:            32 * 1024 * 1024, // default value is 8MiB
 		WriteBuffer:                   16 * 1024 * 1024, // default value is 4MiB
-		CompactionTableSizeMultiplier: 4,
+		CompactionTableSizeMultiplier: 10,
 	}
 
 	if store.db, err = leveldb.OpenFile(dir, opts); err != nil {

From f2c4c888f6f26e632792b1c8db8541051ba02dcf Mon Sep 17 00:00:00 2001
From: Chris Lu <chris.lu@gmail.com>
Date: Fri, 17 May 2019 22:53:30 -0700
Subject: [PATCH 10/10] 1.34

---
 weed/util/constants.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/weed/util/constants.go b/weed/util/constants.go
index 273a60c1d..9706c71b1 100644
--- a/weed/util/constants.go
+++ b/weed/util/constants.go
@@ -5,5 +5,5 @@ import (
 )
 
 var (
-	VERSION = fmt.Sprintf("%s %d.%d", sizeLimit, 1, 33)
+	VERSION = fmt.Sprintf("%s %d.%d", sizeLimit, 1, 34)
 )