diff --git a/weed/filer2/embedded/embedded_store.go b/weed/filer2/embedded/embedded_store.go index d0cfac3f9..5cecbd25c 100644 --- a/weed/filer2/embedded/embedded_store.go +++ b/weed/filer2/embedded/embedded_store.go @@ -26,7 +26,7 @@ func (filer *EmbeddedStore) AddDirectoryLink(directory *filer2.Entry, delta int3 return nil } -func (filer *EmbeddedStore) SetFileChunks(fullpath filer2.FullPath, fileChunks []*filer_pb.FileChunk) (err error) { +func (filer *EmbeddedStore) UpdateEntry(entry *filer2.Entry) (err error) { return nil } diff --git a/weed/filer2/filechunks.go b/weed/filer2/filechunks.go index 714bdab9c..834e014d9 100644 --- a/weed/filer2/filechunks.go +++ b/weed/filer2/filechunks.go @@ -37,6 +37,21 @@ func CompactFileChunks(chunks []*filer_pb.FileChunk) (compacted, garbage []*file return } +func FindUnusedFileChunks(oldChunks, newChunks []*filer_pb.FileChunk) (unused []*filer_pb.FileChunk) { + + fileIds := make(map[string]bool) + for _, interval := range newChunks { + fileIds[interval.FileId] = true + } + for _, chunk := range oldChunks { + if found := fileIds[chunk.FileId]; !found { + unused = append(unused, chunk) + } + } + + return +} + func logPrintf(name string, visibles []*visibleInterval) { // return diff --git a/weed/filer2/filer.go b/weed/filer2/filer.go index facd447f8..6b83db0e6 100644 --- a/weed/filer2/filer.go +++ b/weed/filer2/filer.go @@ -8,7 +8,6 @@ import ( "path/filepath" "time" "os" - "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "github.com/chrislusf/seaweedfs/weed/glog" ) @@ -113,8 +112,8 @@ func (f *Filer) CreateEntry(entry *Entry) (error) { return nil } -func (f *Filer) SetFileChunks(p FullPath, chunks []*filer_pb.FileChunk) (err error) { - return f.store.SetFileChunks(p, chunks) +func (f *Filer) UpdateEntry(entry *Entry) (err error) { + return f.store.UpdateEntry(entry) } func (f *Filer) FindEntry(p FullPath) (found bool, entry *Entry, err error) { diff --git a/weed/filer2/filer_structure.go b/weed/filer2/filer_structure.go index 99ca9ce5a..5e5382d35 100644 --- a/weed/filer2/filer_structure.go +++ b/weed/filer2/filer_structure.go @@ -69,7 +69,7 @@ var ErrNotFound = errors.New("filer: no entry is found in filer store") type FilerStore interface { InsertEntry(*Entry) (error) - SetFileChunks(FullPath, []*filer_pb.FileChunk) (err error) + UpdateEntry(*Entry) (err error) FindEntry(FullPath) (found bool, entry *Entry, err error) DeleteEntry(FullPath) (fileEntry *Entry, err error) diff --git a/weed/filer2/memdb/memdb_store.go b/weed/filer2/memdb/memdb_store.go index 1e888cc45..ba26eb163 100644 --- a/weed/filer2/memdb/memdb_store.go +++ b/weed/filer2/memdb/memdb_store.go @@ -6,7 +6,6 @@ import ( "strings" "fmt" "time" - "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" ) type MemDbStore struct { @@ -29,16 +28,16 @@ func NewMemDbStore() (filer *MemDbStore) { func (filer *MemDbStore) InsertEntry(entry *filer2.Entry) (err error) { // println("inserting", entry.FullPath) + entry.Crtime = time.Now() filer.tree.ReplaceOrInsert(Entry{entry}) return nil } -func (filer *MemDbStore) SetFileChunks(fullpath filer2.FullPath, fileChunks []*filer_pb.FileChunk) (err error) { - found, entry, err := filer.FindEntry(fullpath) +func (filer *MemDbStore) UpdateEntry(entry *filer2.Entry) (err error) { + found, entry, err := filer.FindEntry(entry.FullPath) if !found { - return fmt.Errorf("No such file: %s", fullpath) + return fmt.Errorf("No such file: %s", entry.FullPath) } - entry.Chunks = fileChunks entry.Mtime = time.Now() filer.tree.ReplaceOrInsert(Entry{entry}) return nil diff --git a/weed/filesys/dir.go b/weed/filesys/dir.go index 90c95d4ce..3a0ca9b31 100644 --- a/weed/filesys/dir.go +++ b/weed/filesys/dir.go @@ -76,6 +76,16 @@ func (dir *Dir) Attr(context context.Context, attr *fuse.Attr) error { return nil } +func (dir *Dir) newFile(name string, chunks []*filer_pb.FileChunk) *File { + return &File{ + Name: name, + dir: dir, + wfs: dir.wfs, + // attributes: &filer_pb.FuseAttributes{}, + Chunks: chunks, + } +} + func (dir *Dir) Create(ctx context.Context, req *fuse.CreateRequest, resp *fuse.CreateResponse) (fs.Node, fs.Handle, error) { @@ -104,7 +114,7 @@ func (dir *Dir) Create(ctx context.Context, req *fuse.CreateRequest, }) if err == nil { - node := &File{Name: req.Name, dir: dir, wfs: dir.wfs} + node := dir.newFile(req.Name, nil) dir.NodeMap[req.Name] = node return node, node, nil } @@ -186,7 +196,7 @@ func (dir *Dir) Lookup(ctx context.Context, name string) (node fs.Node, err erro if entry.IsDirectory { node = &Dir{Path: path.Join(dir.Path, name), wfs: dir.wfs} } else { - node = &File{Chunks: entry.Chunks, Name: name, dir: dir, wfs: dir.wfs} + node = dir.newFile(name, entry.Chunks) } dir.NodeMap[name] = node return node, nil diff --git a/weed/filesys/file.go b/weed/filesys/file.go index 81dcba0d9..bbb014bd4 100644 --- a/weed/filesys/file.go +++ b/weed/filesys/file.go @@ -17,7 +17,7 @@ import ( ) var _ = fs.Node(&File{}) -// var _ = fs.NodeOpener(&File{}) +var _ = fs.NodeOpener(&File{}) var _ = fs.NodeFsyncer(&File{}) var _ = fs.Handle(&File{}) var _ = fs.HandleReadAller(&File{}) @@ -28,58 +28,63 @@ var _ = fs.HandleReleaser(&File{}) var _ = fs.NodeSetattrer(&File{}) type File struct { - Chunks []*filer_pb.FileChunk - Name string - dir *Dir - wfs *WFS + Chunks []*filer_pb.FileChunk + Name string + dir *Dir + wfs *WFS + isOpened bool + attributes *filer_pb.FuseAttributes } func (file *File) Attr(context context.Context, attr *fuse.Attr) error { - fullPath := filepath.Join(file.dir.Path, file.Name) - item := file.wfs.listDirectoryEntriesCache.Get(fullPath) - var attributes *filer_pb.FuseAttributes - if item != nil { - attributes = item.Value().(*filer_pb.FuseAttributes) - glog.V(1).Infof("read cached file %v attributes", file.Name) - } else { - err := file.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error { - - request := &filer_pb.GetEntryAttributesRequest{ - Name: file.Name, - ParentDir: file.dir.Path, - } - glog.V(1).Infof("read file size: %v", request) - resp, err := client.GetEntryAttributes(context, request) - if err != nil { - glog.V(0).Infof("read file attributes %v: %v", request, err) - return err - } + if !file.isOpened || file.attributes == nil { + fullPath := filepath.Join(file.dir.Path, file.Name) + item := file.wfs.listDirectoryEntriesCache.Get(fullPath) + if item != nil { + file.attributes = item.Value().(*filer_pb.FuseAttributes) + glog.V(1).Infof("read cached file %v attributes", file.Name) + } else { + err := file.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error { - attributes = resp.Attributes + request := &filer_pb.GetEntryAttributesRequest{ + Name: file.Name, + ParentDir: file.dir.Path, + } - return nil - }) + glog.V(1).Infof("read file size: %v", request) + resp, err := client.GetEntryAttributes(context, request) + if err != nil { + glog.V(0).Infof("read file attributes %v: %v", request, err) + return err + } - if err != nil { - return err + file.attributes = resp.Attributes + + return nil + }) + + if err != nil { + return err + } } } - attr.Mode = os.FileMode(attributes.FileMode) - attr.Size = attributes.FileSize - attr.Mtime = time.Unix(attributes.Mtime, 0) - attr.Gid = attributes.Gid - attr.Uid = attributes.Uid + attr.Mode = os.FileMode(file.attributes.FileMode) + attr.Size = filer2.TotalSize(file.Chunks) + attr.Mtime = time.Unix(file.attributes.Mtime, 0) + attr.Gid = file.attributes.Gid + attr.Uid = file.attributes.Uid return nil } -func (file *File) xOpen(ctx context.Context, req *fuse.OpenRequest, resp *fuse.OpenResponse) (fs.Handle, error) { +func (file *File) Open(ctx context.Context, req *fuse.OpenRequest, resp *fuse.OpenResponse) (fs.Handle, error) { fullPath := filepath.Join(file.dir.Path, file.Name) fmt.Printf("Open %v %+v\n", fullPath, req) + file.isOpened = true return file, nil @@ -89,10 +94,28 @@ func (file *File) Setattr(ctx context.Context, req *fuse.SetattrRequest, resp *f fullPath := filepath.Join(file.dir.Path, file.Name) fmt.Printf("Setattr %v %+v\n", fullPath, req) - if req.Valid.Size() && req.Size == 0 { - fmt.Printf("truncate %v \n", fullPath) - file.Chunks = nil - resp.Attr.Size = 0 + if req.Valid.Size() { + + if req.Size == 0 { + fmt.Printf("truncate %v \n", fullPath) + file.Chunks = nil + } + file.attributes.FileSize = req.Size + } + if req.Valid.Mode() { + file.attributes.FileMode = uint32(req.Mode) + } + + if req.Valid.Uid() { + file.attributes.Uid = req.Uid + } + + if req.Valid.Gid() { + file.attributes.Gid = req.Gid + } + + if req.Valid.Mtime() { + file.attributes.Mtime = req.Mtime.Unix() } return nil @@ -152,16 +175,17 @@ func (file *File) Flush(ctx context.Context, req *fuse.FlushRequest) error { err := file.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error { - request := &filer_pb.SetFileChunksRequest{ + request := &filer_pb.UpdateEntryRequest{ Directory: file.dir.Path, Entry: &filer_pb.Entry{ - Name: file.Name, - Chunks: file.Chunks, + Name: file.Name, + Attributes: file.attributes, + Chunks: file.Chunks, }, } glog.V(1).Infof("append chunks: %v", request) - if _, err := client.SetFileChunks(ctx, request); err != nil { + if _, err := client.UpdateEntry(ctx, request); err != nil { return fmt.Errorf("create file: %v", err) } @@ -224,7 +248,8 @@ func (file *File) Write(ctx context.Context, req *fuse.WriteRequest, resp *fuse. func (file *File) Release(ctx context.Context, req *fuse.ReleaseRequest) error { - // fmt.Printf("release file %+v\n", req) + fmt.Printf("release file %+v\n", req) + file.isOpened = false return nil } diff --git a/weed/pb/filer.proto b/weed/pb/filer.proto index 1f440d19c..e789f9d38 100644 --- a/weed/pb/filer.proto +++ b/weed/pb/filer.proto @@ -21,7 +21,7 @@ service SeaweedFiler { rpc CreateEntry (CreateEntryRequest) returns (CreateEntryResponse) { } - rpc SetFileChunks (SetFileChunksRequest) returns (SetFileChunksResponse) { + rpc UpdateEntry (UpdateEntryRequest) returns (UpdateEntryResponse) { } rpc DeleteEntry (DeleteEntryRequest) returns (DeleteEntryResponse) { @@ -121,9 +121,9 @@ message AssignVolumeResponse { int32 count = 4; } -message SetFileChunksRequest { +message UpdateEntryRequest { string directory = 1; Entry entry = 2; } -message SetFileChunksResponse { +message UpdateEntryResponse { } diff --git a/weed/pb/filer_pb/filer.pb.go b/weed/pb/filer_pb/filer.pb.go index a8a20ffe6..7ef1c294b 100644 --- a/weed/pb/filer_pb/filer.pb.go +++ b/weed/pb/filer_pb/filer.pb.go @@ -26,8 +26,8 @@ It has these top-level messages: DeleteEntryResponse AssignVolumeRequest AssignVolumeResponse - SetFileChunksRequest - SetFileChunksResponse + UpdateEntryRequest + UpdateEntryResponse */ package filer_pb @@ -475,37 +475,37 @@ func (m *AssignVolumeResponse) GetCount() int32 { return 0 } -type SetFileChunksRequest struct { +type UpdateEntryRequest struct { Directory string `protobuf:"bytes,1,opt,name=directory" json:"directory,omitempty"` Entry *Entry `protobuf:"bytes,2,opt,name=entry" json:"entry,omitempty"` } -func (m *SetFileChunksRequest) Reset() { *m = SetFileChunksRequest{} } -func (m *SetFileChunksRequest) String() string { return proto.CompactTextString(m) } -func (*SetFileChunksRequest) ProtoMessage() {} -func (*SetFileChunksRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{17} } +func (m *UpdateEntryRequest) Reset() { *m = UpdateEntryRequest{} } +func (m *UpdateEntryRequest) String() string { return proto.CompactTextString(m) } +func (*UpdateEntryRequest) ProtoMessage() {} +func (*UpdateEntryRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{17} } -func (m *SetFileChunksRequest) GetDirectory() string { +func (m *UpdateEntryRequest) GetDirectory() string { if m != nil { return m.Directory } return "" } -func (m *SetFileChunksRequest) GetEntry() *Entry { +func (m *UpdateEntryRequest) GetEntry() *Entry { if m != nil { return m.Entry } return nil } -type SetFileChunksResponse struct { +type UpdateEntryResponse struct { } -func (m *SetFileChunksResponse) Reset() { *m = SetFileChunksResponse{} } -func (m *SetFileChunksResponse) String() string { return proto.CompactTextString(m) } -func (*SetFileChunksResponse) ProtoMessage() {} -func (*SetFileChunksResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{18} } +func (m *UpdateEntryResponse) Reset() { *m = UpdateEntryResponse{} } +func (m *UpdateEntryResponse) String() string { return proto.CompactTextString(m) } +func (*UpdateEntryResponse) ProtoMessage() {} +func (*UpdateEntryResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{18} } func init() { proto.RegisterType((*LookupDirectoryEntryRequest)(nil), "filer_pb.LookupDirectoryEntryRequest") @@ -525,8 +525,8 @@ func init() { proto.RegisterType((*DeleteEntryResponse)(nil), "filer_pb.DeleteEntryResponse") proto.RegisterType((*AssignVolumeRequest)(nil), "filer_pb.AssignVolumeRequest") proto.RegisterType((*AssignVolumeResponse)(nil), "filer_pb.AssignVolumeResponse") - proto.RegisterType((*SetFileChunksRequest)(nil), "filer_pb.SetFileChunksRequest") - proto.RegisterType((*SetFileChunksResponse)(nil), "filer_pb.SetFileChunksResponse") + proto.RegisterType((*UpdateEntryRequest)(nil), "filer_pb.UpdateEntryRequest") + proto.RegisterType((*UpdateEntryResponse)(nil), "filer_pb.UpdateEntryResponse") } // Reference imports to suppress errors if they are not otherwise used. @@ -545,7 +545,7 @@ type SeaweedFilerClient interface { GetEntryAttributes(ctx context.Context, in *GetEntryAttributesRequest, opts ...grpc.CallOption) (*GetEntryAttributesResponse, error) GetFileContent(ctx context.Context, in *GetFileContentRequest, opts ...grpc.CallOption) (*GetFileContentResponse, error) CreateEntry(ctx context.Context, in *CreateEntryRequest, opts ...grpc.CallOption) (*CreateEntryResponse, error) - SetFileChunks(ctx context.Context, in *SetFileChunksRequest, opts ...grpc.CallOption) (*SetFileChunksResponse, error) + UpdateEntry(ctx context.Context, in *UpdateEntryRequest, opts ...grpc.CallOption) (*UpdateEntryResponse, error) DeleteEntry(ctx context.Context, in *DeleteEntryRequest, opts ...grpc.CallOption) (*DeleteEntryResponse, error) AssignVolume(ctx context.Context, in *AssignVolumeRequest, opts ...grpc.CallOption) (*AssignVolumeResponse, error) } @@ -603,9 +603,9 @@ func (c *seaweedFilerClient) CreateEntry(ctx context.Context, in *CreateEntryReq return out, nil } -func (c *seaweedFilerClient) SetFileChunks(ctx context.Context, in *SetFileChunksRequest, opts ...grpc.CallOption) (*SetFileChunksResponse, error) { - out := new(SetFileChunksResponse) - err := grpc.Invoke(ctx, "/filer_pb.SeaweedFiler/SetFileChunks", in, out, c.cc, opts...) +func (c *seaweedFilerClient) UpdateEntry(ctx context.Context, in *UpdateEntryRequest, opts ...grpc.CallOption) (*UpdateEntryResponse, error) { + out := new(UpdateEntryResponse) + err := grpc.Invoke(ctx, "/filer_pb.SeaweedFiler/UpdateEntry", in, out, c.cc, opts...) if err != nil { return nil, err } @@ -638,7 +638,7 @@ type SeaweedFilerServer interface { GetEntryAttributes(context.Context, *GetEntryAttributesRequest) (*GetEntryAttributesResponse, error) GetFileContent(context.Context, *GetFileContentRequest) (*GetFileContentResponse, error) CreateEntry(context.Context, *CreateEntryRequest) (*CreateEntryResponse, error) - SetFileChunks(context.Context, *SetFileChunksRequest) (*SetFileChunksResponse, error) + UpdateEntry(context.Context, *UpdateEntryRequest) (*UpdateEntryResponse, error) DeleteEntry(context.Context, *DeleteEntryRequest) (*DeleteEntryResponse, error) AssignVolume(context.Context, *AssignVolumeRequest) (*AssignVolumeResponse, error) } @@ -737,20 +737,20 @@ func _SeaweedFiler_CreateEntry_Handler(srv interface{}, ctx context.Context, dec return interceptor(ctx, in, info, handler) } -func _SeaweedFiler_SetFileChunks_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(SetFileChunksRequest) +func _SeaweedFiler_UpdateEntry_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(UpdateEntryRequest) if err := dec(in); err != nil { return nil, err } if interceptor == nil { - return srv.(SeaweedFilerServer).SetFileChunks(ctx, in) + return srv.(SeaweedFilerServer).UpdateEntry(ctx, in) } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/filer_pb.SeaweedFiler/SetFileChunks", + FullMethod: "/filer_pb.SeaweedFiler/UpdateEntry", } handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(SeaweedFilerServer).SetFileChunks(ctx, req.(*SetFileChunksRequest)) + return srv.(SeaweedFilerServer).UpdateEntry(ctx, req.(*UpdateEntryRequest)) } return interceptor(ctx, in, info, handler) } @@ -816,8 +816,8 @@ var _SeaweedFiler_serviceDesc = grpc.ServiceDesc{ Handler: _SeaweedFiler_CreateEntry_Handler, }, { - MethodName: "SetFileChunks", - Handler: _SeaweedFiler_SetFileChunks_Handler, + MethodName: "UpdateEntry", + Handler: _SeaweedFiler_UpdateEntry_Handler, }, { MethodName: "DeleteEntry", @@ -835,53 +835,53 @@ var _SeaweedFiler_serviceDesc = grpc.ServiceDesc{ func init() { proto.RegisterFile("filer.proto", fileDescriptor0) } var fileDescriptor0 = []byte{ - // 765 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xac, 0x56, 0xdd, 0x4e, 0xdb, 0x4a, - 0x10, 0xc6, 0x38, 0x09, 0x64, 0x12, 0x38, 0x47, 0x93, 0x00, 0x3e, 0xe1, 0x2f, 0xc7, 0x2d, 0x15, - 0x55, 0x25, 0x54, 0xa5, 0x37, 0xbd, 0x2c, 0x02, 0x8a, 0x2a, 0x51, 0x21, 0x19, 0x81, 0x54, 0xf5, - 0x22, 0x4a, 0xec, 0x49, 0xba, 0xc2, 0xb1, 0x53, 0x7b, 0xdd, 0x8a, 0x5e, 0xf7, 0x55, 0xfa, 0x1e, - 0x7d, 0xb4, 0x6a, 0xd7, 0x1b, 0x7b, 0x4d, 0x1c, 0x5a, 0xa4, 0xde, 0x79, 0xe7, 0xef, 0xfb, 0x76, - 0x66, 0xbe, 0x4d, 0xa0, 0x31, 0x62, 0x3e, 0x45, 0x47, 0xd3, 0x28, 0xe4, 0x21, 0xae, 0xca, 0x43, - 0x7f, 0x3a, 0xb4, 0x2f, 0x61, 0xfb, 0x22, 0x0c, 0x6f, 0x93, 0xe9, 0x29, 0x8b, 0xc8, 0xe5, 0x61, - 0x74, 0x77, 0x16, 0xf0, 0xe8, 0xce, 0xa1, 0xcf, 0x09, 0xc5, 0x1c, 0x77, 0xa0, 0xee, 0xcd, 0x1c, - 0x96, 0xd1, 0x35, 0x0e, 0xeb, 0x4e, 0x6e, 0x40, 0x84, 0x4a, 0x30, 0x98, 0x90, 0xb5, 0x2c, 0x1d, - 0xf2, 0xdb, 0x3e, 0x83, 0x9d, 0xf2, 0x82, 0xf1, 0x34, 0x0c, 0x62, 0xc2, 0x03, 0xa8, 0x92, 0x30, - 0xc8, 0x6a, 0x8d, 0xde, 0x3f, 0x47, 0x33, 0x2a, 0x47, 0x69, 0x5c, 0xea, 0xb5, 0x7b, 0x80, 0x17, - 0x2c, 0xe6, 0xc2, 0xc6, 0x28, 0xfe, 0x23, 0x3a, 0xf6, 0x1b, 0x68, 0x15, 0x72, 0x14, 0xe2, 0x73, - 0x58, 0xa1, 0xd4, 0x64, 0x19, 0x5d, 0xb3, 0x0c, 0x73, 0xe6, 0xb7, 0x7f, 0x18, 0x50, 0x95, 0xa6, - 0xec, 0x6a, 0x46, 0x7e, 0x35, 0xfc, 0x1f, 0x9a, 0x2c, 0xee, 0xe7, 0x04, 0xc4, 0xb5, 0x57, 0x9d, - 0x06, 0x8b, 0xb3, 0xab, 0xe2, 0x0b, 0xa8, 0xb9, 0x9f, 0x92, 0xe0, 0x36, 0xb6, 0x4c, 0x09, 0xd5, - 0xca, 0xa1, 0xde, 0x32, 0x9f, 0x4e, 0x84, 0xcf, 0x51, 0x21, 0xf8, 0x1a, 0x60, 0xc0, 0x79, 0xc4, - 0x86, 0x09, 0xa7, 0xd8, 0xaa, 0xc8, 0x7e, 0x58, 0x5a, 0x42, 0x12, 0xd3, 0x71, 0xe6, 0x77, 0xb4, - 0x58, 0x7b, 0x04, 0xf5, 0xac, 0x1c, 0x6e, 0xc1, 0x8a, 0xc8, 0xe9, 0x33, 0x4f, 0xb1, 0xad, 0x89, - 0xe3, 0x3b, 0x0f, 0x37, 0xa1, 0x16, 0x8e, 0x46, 0x31, 0x71, 0xc9, 0xd4, 0x74, 0xd4, 0x49, 0xdc, - 0x2d, 0x66, 0xdf, 0xc8, 0x32, 0xbb, 0xc6, 0x61, 0xc5, 0x91, 0xdf, 0xd8, 0x86, 0xea, 0x84, 0xb3, - 0x09, 0x49, 0x1a, 0xa6, 0x93, 0x1e, 0xec, 0xef, 0x06, 0xac, 0x17, 0x69, 0xe0, 0x36, 0xd4, 0x25, - 0x9a, 0xac, 0x60, 0xc8, 0x0a, 0x72, 0x9b, 0xae, 0x0a, 0x55, 0x96, 0xb5, 0x2a, 0x59, 0xca, 0x24, - 0xf4, 0x52, 0xd0, 0xb5, 0x34, 0xe5, 0x7d, 0xe8, 0x11, 0xfe, 0x0b, 0x66, 0xc2, 0x3c, 0x09, 0xbb, - 0xe6, 0x88, 0x4f, 0x61, 0x19, 0x33, 0xcf, 0xaa, 0xa6, 0x96, 0x31, 0xf3, 0xec, 0x31, 0xfc, 0x77, - 0x4e, 0x72, 0xae, 0x77, 0x5a, 0x43, 0xd4, 0x4e, 0x94, 0x4d, 0x6a, 0x17, 0x60, 0x3a, 0x88, 0x28, - 0xe0, 0x62, 0x5a, 0x6a, 0x3d, 0xeb, 0xa9, 0xe5, 0x94, 0x45, 0x7a, 0xc7, 0x4c, 0xbd, 0x63, 0xf6, - 0x0d, 0x74, 0xca, 0x80, 0xd4, 0x22, 0x15, 0xe7, 0x65, 0x3c, 0x62, 0x5e, 0x2f, 0x61, 0xe3, 0x9c, - 0xb8, 0x1c, 0x59, 0x18, 0x70, 0x0a, 0xf8, 0x8c, 0xfc, 0xa2, 0xd9, 0xd9, 0x3d, 0xd8, 0xbc, 0x9f, - 0xa1, 0x58, 0x58, 0xb0, 0xe2, 0xa6, 0x26, 0x99, 0xd2, 0x74, 0x66, 0x47, 0xfb, 0x03, 0xe0, 0x49, - 0x44, 0x03, 0x4e, 0x8f, 0x90, 0x70, 0x26, 0xc7, 0xe5, 0x07, 0xe5, 0xb8, 0x01, 0xad, 0x42, 0xe9, - 0x94, 0x8b, 0xcd, 0x00, 0x4f, 0xc9, 0xa7, 0x47, 0x21, 0x96, 0x3c, 0x1a, 0x73, 0xca, 0x32, 0xe7, - 0x94, 0x25, 0x18, 0x14, 0xa0, 0x14, 0x83, 0x09, 0xb4, 0x8e, 0xe3, 0x98, 0x8d, 0x83, 0x9b, 0xd0, - 0x4f, 0x26, 0x34, 0xa3, 0xd0, 0x86, 0xaa, 0x1b, 0x26, 0xaa, 0x45, 0x55, 0x27, 0x3d, 0xe0, 0x1e, - 0x80, 0x1b, 0xfa, 0x3e, 0xb9, 0x9c, 0x85, 0x81, 0x22, 0xa0, 0x59, 0xb0, 0x0b, 0x8d, 0x88, 0xa6, - 0x3e, 0x73, 0x07, 0x32, 0x20, 0xdd, 0x0d, 0xdd, 0x64, 0x7f, 0x81, 0x76, 0x11, 0x4e, 0x0d, 0x65, - 0xa1, 0x06, 0xc5, 0x7a, 0x47, 0xbe, 0xc2, 0x12, 0x9f, 0x72, 0x37, 0x93, 0xa1, 0xcf, 0xdc, 0xbe, - 0x70, 0x98, 0x6a, 0x37, 0xa5, 0xe5, 0x3a, 0xf2, 0x73, 0xe6, 0x15, 0x8d, 0xb9, 0xfd, 0x11, 0xda, - 0x57, 0x6a, 0x1d, 0xe4, 0xdb, 0xf1, 0x57, 0x87, 0xbb, 0x05, 0x1b, 0xf7, 0x8a, 0xa7, 0xb7, 0xea, - 0xfd, 0xac, 0x42, 0xf3, 0x8a, 0x06, 0x5f, 0x89, 0x3c, 0xe1, 0x8d, 0x70, 0x0c, 0xed, 0xb2, 0xc7, - 0x1d, 0x0f, 0xf2, 0xca, 0x0f, 0xfc, 0x9a, 0x74, 0x9e, 0xfd, 0x2e, 0x4c, 0x0d, 0x75, 0x09, 0x2f, - 0xa0, 0xa1, 0x3d, 0xe5, 0xb8, 0xa3, 0x25, 0xce, 0xfd, 0x2a, 0x74, 0x76, 0x17, 0x78, 0xb3, 0x6a, - 0x03, 0xc0, 0x79, 0x59, 0xe3, 0x93, 0x3c, 0x6d, 0xe1, 0xeb, 0xd2, 0x79, 0xfa, 0x70, 0x50, 0x06, - 0x71, 0x0d, 0xeb, 0x45, 0xbd, 0xe2, 0x7e, 0x21, 0x73, 0x5e, 0xfb, 0x9d, 0xee, 0xe2, 0x00, 0xbd, - 0x0f, 0x9a, 0xee, 0xf4, 0x3e, 0xcc, 0x2b, 0x5d, 0xef, 0x43, 0x99, 0x58, 0x97, 0xd0, 0x81, 0xb5, - 0xc2, 0xa0, 0x71, 0x2f, 0xcf, 0x28, 0x5b, 0xaf, 0xce, 0xfe, 0x42, 0xbf, 0xce, 0x50, 0xd3, 0xa5, - 0xce, 0x70, 0xfe, 0x65, 0xd0, 0x19, 0x96, 0x89, 0x79, 0x09, 0x2f, 0xa1, 0xa9, 0xeb, 0x0b, 0xb5, - 0x84, 0x12, 0x99, 0x77, 0xf6, 0x16, 0xb9, 0x67, 0x05, 0x87, 0x35, 0xf9, 0x87, 0xe7, 0xd5, 0xaf, - 0x00, 0x00, 0x00, 0xff, 0xff, 0xc7, 0x51, 0xe7, 0x2b, 0xff, 0x08, 0x00, 0x00, + // 754 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xb4, 0x56, 0xdd, 0x6e, 0xd3, 0x4a, + 0x10, 0xae, 0xeb, 0x24, 0x6d, 0x26, 0x69, 0xcf, 0xd1, 0x26, 0xed, 0xf1, 0x49, 0x7f, 0x08, 0x86, + 0xa2, 0x22, 0xa4, 0x0a, 0x85, 0x1b, 0x2e, 0xa9, 0xda, 0x52, 0x21, 0x15, 0x55, 0x72, 0xd5, 0x4a, + 0x5c, 0x45, 0x89, 0x3d, 0x09, 0xab, 0x3a, 0x76, 0xf0, 0xae, 0x41, 0xe5, 0x9a, 0x57, 0xe1, 0x25, + 0x78, 0x3a, 0xb4, 0x3f, 0xb1, 0xd7, 0xd8, 0x29, 0xf4, 0x82, 0x3b, 0xef, 0xfc, 0x7c, 0xf3, 0xed, + 0xcc, 0x7c, 0x9b, 0x40, 0x6b, 0x42, 0x43, 0x4c, 0x8e, 0xe6, 0x49, 0xcc, 0x63, 0xb2, 0x2e, 0x0f, + 0xc3, 0xf9, 0xd8, 0xbd, 0x84, 0x9d, 0x8b, 0x38, 0xbe, 0x4d, 0xe7, 0xa7, 0x34, 0x41, 0x9f, 0xc7, + 0xc9, 0xdd, 0x59, 0xc4, 0x93, 0x3b, 0x0f, 0x3f, 0xa5, 0xc8, 0x38, 0xd9, 0x85, 0x66, 0xb0, 0x70, + 0x38, 0x56, 0xdf, 0x3a, 0x6c, 0x7a, 0xb9, 0x81, 0x10, 0xa8, 0x45, 0xa3, 0x19, 0x3a, 0xab, 0xd2, + 0x21, 0xbf, 0xdd, 0x33, 0xd8, 0xad, 0x06, 0x64, 0xf3, 0x38, 0x62, 0x48, 0x0e, 0xa0, 0x8e, 0xc2, + 0x20, 0xd1, 0x5a, 0x83, 0x7f, 0x8e, 0x16, 0x54, 0x8e, 0x54, 0x9c, 0xf2, 0xba, 0x03, 0x20, 0x17, + 0x94, 0x71, 0x61, 0xa3, 0xc8, 0xfe, 0x88, 0x8e, 0xfb, 0x06, 0x3a, 0x85, 0x1c, 0x5d, 0xf1, 0x39, + 0xac, 0xa1, 0x32, 0x39, 0x56, 0xdf, 0xae, 0xaa, 0xb9, 0xf0, 0xbb, 0xdf, 0x2d, 0xa8, 0x4b, 0x53, + 0x76, 0x35, 0x2b, 0xbf, 0x1a, 0x79, 0x0c, 0x6d, 0xca, 0x86, 0x39, 0x01, 0x71, 0xed, 0x75, 0xaf, + 0x45, 0x59, 0x76, 0x55, 0xf2, 0x02, 0x1a, 0xfe, 0xc7, 0x34, 0xba, 0x65, 0x8e, 0x2d, 0x4b, 0x75, + 0xf2, 0x52, 0x6f, 0x69, 0x88, 0x27, 0xc2, 0xe7, 0xe9, 0x10, 0xf2, 0x1a, 0x60, 0xc4, 0x79, 0x42, + 0xc7, 0x29, 0x47, 0xe6, 0xd4, 0x64, 0x3f, 0x1c, 0x23, 0x21, 0x65, 0x78, 0x9c, 0xf9, 0x3d, 0x23, + 0xd6, 0x9d, 0x40, 0x33, 0x83, 0x23, 0xff, 0xc1, 0x9a, 0xc8, 0x19, 0xd2, 0x40, 0xb3, 0x6d, 0x88, + 0xe3, 0xbb, 0x80, 0x6c, 0x43, 0x23, 0x9e, 0x4c, 0x18, 0x72, 0xc9, 0xd4, 0xf6, 0xf4, 0x49, 0xdc, + 0x8d, 0xd1, 0xaf, 0xe8, 0xd8, 0x7d, 0xeb, 0xb0, 0xe6, 0xc9, 0x6f, 0xd2, 0x85, 0xfa, 0x8c, 0xd3, + 0x19, 0x4a, 0x1a, 0xb6, 0xa7, 0x0e, 0xee, 0x37, 0x0b, 0x36, 0x8b, 0x34, 0xc8, 0x0e, 0x34, 0x65, + 0x35, 0x89, 0x60, 0x49, 0x04, 0xb9, 0x4d, 0x57, 0x05, 0x94, 0x55, 0x03, 0x25, 0x4b, 0x99, 0xc5, + 0x81, 0x2a, 0xba, 0xa1, 0x52, 0xde, 0xc7, 0x01, 0x92, 0x7f, 0xc1, 0x4e, 0x69, 0x20, 0xcb, 0x6e, + 0x78, 0xe2, 0x53, 0x58, 0xa6, 0x34, 0x70, 0xea, 0xca, 0x32, 0xa5, 0x81, 0x3b, 0x85, 0xff, 0xcf, + 0x51, 0xce, 0xf5, 0xce, 0x68, 0x88, 0xde, 0x89, 0xaa, 0x49, 0xed, 0x01, 0xcc, 0x47, 0x09, 0x46, + 0x5c, 0x4c, 0x4b, 0xaf, 0x67, 0x53, 0x59, 0x4e, 0x69, 0x62, 0x76, 0xcc, 0x36, 0x3b, 0xe6, 0xde, + 0x40, 0xaf, 0xaa, 0x90, 0x5e, 0xa4, 0xe2, 0xbc, 0xac, 0x07, 0xcc, 0xeb, 0x25, 0x6c, 0x9d, 0x23, + 0x97, 0x23, 0x8b, 0x23, 0x8e, 0x11, 0x5f, 0x90, 0x5f, 0x36, 0x3b, 0x77, 0x00, 0xdb, 0xbf, 0x66, + 0x68, 0x16, 0x0e, 0xac, 0xf9, 0xca, 0x24, 0x53, 0xda, 0xde, 0xe2, 0xe8, 0x7e, 0x00, 0x72, 0x92, + 0xe0, 0x88, 0xe3, 0x03, 0x24, 0x9c, 0xc9, 0x71, 0xf5, 0x5e, 0x39, 0x6e, 0x41, 0xa7, 0x00, 0xad, + 0xb8, 0xb8, 0x14, 0xc8, 0x29, 0x86, 0xf8, 0xa0, 0x8a, 0x15, 0x8f, 0x46, 0x49, 0x59, 0x76, 0x49, + 0x59, 0x82, 0x41, 0xa1, 0x94, 0x66, 0x30, 0x83, 0xce, 0x31, 0x63, 0x74, 0x1a, 0xdd, 0xc4, 0x61, + 0x3a, 0xc3, 0x05, 0x85, 0x2e, 0xd4, 0xfd, 0x38, 0xd5, 0x2d, 0xaa, 0x7b, 0xea, 0x40, 0xf6, 0x01, + 0xfc, 0x38, 0x0c, 0xd1, 0xe7, 0x34, 0x8e, 0x34, 0x01, 0xc3, 0x42, 0xfa, 0xd0, 0x4a, 0x70, 0x1e, + 0x52, 0x7f, 0x24, 0x03, 0xd4, 0x6e, 0x98, 0x26, 0xf7, 0x33, 0x74, 0x8b, 0xe5, 0xf4, 0x50, 0x96, + 0x6a, 0x50, 0xac, 0x77, 0x12, 0xea, 0x5a, 0xe2, 0x53, 0xee, 0x66, 0x3a, 0x0e, 0xa9, 0x3f, 0x14, + 0x0e, 0x5b, 0xef, 0xa6, 0xb4, 0x5c, 0x27, 0x61, 0xce, 0xbc, 0x66, 0x30, 0x17, 0xa3, 0xbd, 0x9e, + 0x07, 0x7f, 0x6b, 0xb4, 0x05, 0x68, 0x75, 0xa3, 0xc1, 0x8f, 0x3a, 0xb4, 0xaf, 0x70, 0xf4, 0x05, + 0x31, 0x10, 0x5b, 0x98, 0x90, 0x29, 0x74, 0xab, 0x1e, 0x76, 0x72, 0x90, 0xe3, 0xde, 0xf3, 0x4b, + 0xd2, 0x7b, 0xf6, 0xbb, 0x30, 0x3d, 0xd0, 0x15, 0x72, 0x01, 0x2d, 0xe3, 0x19, 0x27, 0xbb, 0x46, + 0x62, 0xe9, 0x17, 0xa1, 0xb7, 0xb7, 0xc4, 0x9b, 0xa1, 0x8d, 0x80, 0x94, 0x25, 0x4d, 0x9e, 0xe4, + 0x69, 0x4b, 0x5f, 0x96, 0xde, 0xd3, 0xfb, 0x83, 0xb2, 0x12, 0xd7, 0xb0, 0x59, 0xd4, 0x2a, 0x79, + 0x54, 0xc8, 0x2c, 0xeb, 0xbe, 0xd7, 0x5f, 0x1e, 0x60, 0xf6, 0xc1, 0xd0, 0x9c, 0xd9, 0x87, 0xb2, + 0xca, 0xcd, 0x3e, 0x54, 0x09, 0x55, 0xa2, 0x19, 0x63, 0x36, 0xd1, 0xca, 0x8b, 0x65, 0xa2, 0x55, + 0xec, 0x86, 0x42, 0x33, 0xd4, 0x68, 0xa2, 0x95, 0xdf, 0x03, 0x13, 0xad, 0x4a, 0xc2, 0x2b, 0xe4, + 0x12, 0xda, 0xa6, 0xaa, 0x88, 0x91, 0x50, 0x21, 0xee, 0xde, 0xfe, 0x32, 0xf7, 0x02, 0x70, 0xdc, + 0x90, 0x7f, 0x73, 0x5e, 0xfd, 0x0c, 0x00, 0x00, 0xff, 0xff, 0x9c, 0xf1, 0x24, 0x43, 0xf5, 0x08, + 0x00, 0x00, } diff --git a/weed/server/filer_grpc_server.go b/weed/server/filer_grpc_server.go index 3f05d777b..8d643ed2d 100644 --- a/weed/server/filer_grpc_server.go +++ b/weed/server/filer_grpc_server.go @@ -120,34 +120,46 @@ func (fs *FilerServer) CreateEntry(ctx context.Context, req *filer_pb.CreateEntr return &filer_pb.CreateEntryResponse{}, err } -func (fs *FilerServer) SetFileChunks(ctx context.Context, req *filer_pb.SetFileChunksRequest) (*filer_pb.SetFileChunksResponse, error) { +func (fs *FilerServer) UpdateEntry(ctx context.Context, req *filer_pb.UpdateEntryRequest) (*filer_pb.UpdateEntryResponse, error) { fullpath := filepath.Join(req.Directory, req.Entry.Name) found, entry, err := fs.filer.FindEntry(filer2.FullPath(fullpath)) if err != nil { - return &filer_pb.SetFileChunksResponse{}, err + return &filer_pb.UpdateEntryResponse{}, err } if !found { - return &filer_pb.SetFileChunksResponse{}, fmt.Errorf("file not found: %s", fullpath) + return &filer_pb.UpdateEntryResponse{}, fmt.Errorf("file not found: %s", fullpath) } - chunks := append(entry.Chunks, req.Entry.Chunks...) + // remove old chunks if not included in the new ones + unusedChunks := filer2.FindUnusedFileChunks(entry.Chunks, req.Entry.Chunks) - chunks, garbages := filer2.CompactFileChunks(chunks) + chunks, garbages := filer2.CompactFileChunks(req.Entry.Chunks) - err = fs.filer.SetFileChunks( - filer2.FullPath(fullpath), - chunks, - ) + err = fs.filer.UpdateEntry(&filer2.Entry{ + FullPath: filer2.FullPath(filepath.Join(req.Directory, req.Entry.Name)), + Attr: filer2.Attr{ + Mtime: time.Unix(req.Entry.Attributes.Mtime, 0), + Crtime: time.Unix(req.Entry.Attributes.Mtime, 0), + Mode: os.FileMode(req.Entry.Attributes.FileMode), + Uid: req.Entry.Attributes.Uid, + Gid: req.Entry.Attributes.Gid, + }, + Chunks: chunks, + }) if err == nil { - for _, garbage := range garbages { + for _, garbage := range unusedChunks { glog.V(0).Infof("deleting %s old chunk: %v, [%d, %d)", fullpath, garbage.FileId, garbage.Offset, garbage.Offset+int64(garbage.Size)) operation.DeleteFile(fs.master, garbage.FileId, fs.jwt(garbage.FileId)) } + for _, garbage := range garbages { + glog.V(0).Infof("deleting %s garbage chunk: %v, [%d, %d)", fullpath, garbage.FileId, garbage.Offset, garbage.Offset+int64(garbage.Size)) + operation.DeleteFile(fs.master, garbage.FileId, fs.jwt(garbage.FileId)) + } } - return &filer_pb.SetFileChunksResponse{}, err + return &filer_pb.UpdateEntryResponse{}, err } func (fs *FilerServer) DeleteEntry(ctx context.Context, req *filer_pb.DeleteEntryRequest) (resp *filer_pb.DeleteEntryResponse, err error) {