Browse Source

weed mount can work well

TODO: somehow filer url is returning empty content
pull/664/head
Chris Lu 7 years ago
parent
commit
7362de9a18
  1. 2
      weed/filer2/embedded/embedded_store.go
  2. 15
      weed/filer2/filechunks.go
  3. 5
      weed/filer2/filer.go
  4. 2
      weed/filer2/filer_structure.go
  5. 9
      weed/filer2/memdb/memdb_store.go
  6. 14
      weed/filesys/dir.go
  7. 113
      weed/filesys/file.go
  8. 6
      weed/pb/filer.proto
  9. 154
      weed/pb/filer_pb/filer.pb.go
  10. 34
      weed/server/filer_grpc_server.go

2
weed/filer2/embedded/embedded_store.go

@ -26,7 +26,7 @@ func (filer *EmbeddedStore) AddDirectoryLink(directory *filer2.Entry, delta int3
return nil return nil
} }
func (filer *EmbeddedStore) SetFileChunks(fullpath filer2.FullPath, fileChunks []*filer_pb.FileChunk) (err error) {
func (filer *EmbeddedStore) UpdateEntry(entry *filer2.Entry) (err error) {
return nil return nil
} }

15
weed/filer2/filechunks.go

@ -37,6 +37,21 @@ func CompactFileChunks(chunks []*filer_pb.FileChunk) (compacted, garbage []*file
return return
} }
func FindUnusedFileChunks(oldChunks, newChunks []*filer_pb.FileChunk) (unused []*filer_pb.FileChunk) {
fileIds := make(map[string]bool)
for _, interval := range newChunks {
fileIds[interval.FileId] = true
}
for _, chunk := range oldChunks {
if found := fileIds[chunk.FileId]; !found {
unused = append(unused, chunk)
}
}
return
}
func logPrintf(name string, visibles []*visibleInterval) { func logPrintf(name string, visibles []*visibleInterval) {
// return // return

5
weed/filer2/filer.go

@ -8,7 +8,6 @@ import (
"path/filepath" "path/filepath"
"time" "time"
"os" "os"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/glog"
) )
@ -113,8 +112,8 @@ func (f *Filer) CreateEntry(entry *Entry) (error) {
return nil return nil
} }
func (f *Filer) SetFileChunks(p FullPath, chunks []*filer_pb.FileChunk) (err error) {
return f.store.SetFileChunks(p, chunks)
func (f *Filer) UpdateEntry(entry *Entry) (err error) {
return f.store.UpdateEntry(entry)
} }
func (f *Filer) FindEntry(p FullPath) (found bool, entry *Entry, err error) { func (f *Filer) FindEntry(p FullPath) (found bool, entry *Entry, err error) {

2
weed/filer2/filer_structure.go

@ -69,7 +69,7 @@ var ErrNotFound = errors.New("filer: no entry is found in filer store")
type FilerStore interface { type FilerStore interface {
InsertEntry(*Entry) (error) InsertEntry(*Entry) (error)
SetFileChunks(FullPath, []*filer_pb.FileChunk) (err error)
UpdateEntry(*Entry) (err error)
FindEntry(FullPath) (found bool, entry *Entry, err error) FindEntry(FullPath) (found bool, entry *Entry, err error)
DeleteEntry(FullPath) (fileEntry *Entry, err error) DeleteEntry(FullPath) (fileEntry *Entry, err error)

9
weed/filer2/memdb/memdb_store.go

@ -6,7 +6,6 @@ import (
"strings" "strings"
"fmt" "fmt"
"time" "time"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
) )
type MemDbStore struct { type MemDbStore struct {
@ -29,16 +28,16 @@ func NewMemDbStore() (filer *MemDbStore) {
func (filer *MemDbStore) InsertEntry(entry *filer2.Entry) (err error) { func (filer *MemDbStore) InsertEntry(entry *filer2.Entry) (err error) {
// println("inserting", entry.FullPath) // println("inserting", entry.FullPath)
entry.Crtime = time.Now()
filer.tree.ReplaceOrInsert(Entry{entry}) filer.tree.ReplaceOrInsert(Entry{entry})
return nil return nil
} }
func (filer *MemDbStore) SetFileChunks(fullpath filer2.FullPath, fileChunks []*filer_pb.FileChunk) (err error) {
found, entry, err := filer.FindEntry(fullpath)
func (filer *MemDbStore) UpdateEntry(entry *filer2.Entry) (err error) {
found, entry, err := filer.FindEntry(entry.FullPath)
if !found { if !found {
return fmt.Errorf("No such file: %s", fullpath)
return fmt.Errorf("No such file: %s", entry.FullPath)
} }
entry.Chunks = fileChunks
entry.Mtime = time.Now() entry.Mtime = time.Now()
filer.tree.ReplaceOrInsert(Entry{entry}) filer.tree.ReplaceOrInsert(Entry{entry})
return nil return nil

14
weed/filesys/dir.go

@ -76,6 +76,16 @@ func (dir *Dir) Attr(context context.Context, attr *fuse.Attr) error {
return nil return nil
} }
func (dir *Dir) newFile(name string, chunks []*filer_pb.FileChunk) *File {
return &File{
Name: name,
dir: dir,
wfs: dir.wfs,
// attributes: &filer_pb.FuseAttributes{},
Chunks: chunks,
}
}
func (dir *Dir) Create(ctx context.Context, req *fuse.CreateRequest, func (dir *Dir) Create(ctx context.Context, req *fuse.CreateRequest,
resp *fuse.CreateResponse) (fs.Node, fs.Handle, error) { resp *fuse.CreateResponse) (fs.Node, fs.Handle, error) {
@ -104,7 +114,7 @@ func (dir *Dir) Create(ctx context.Context, req *fuse.CreateRequest,
}) })
if err == nil { if err == nil {
node := &File{Name: req.Name, dir: dir, wfs: dir.wfs}
node := dir.newFile(req.Name, nil)
dir.NodeMap[req.Name] = node dir.NodeMap[req.Name] = node
return node, node, nil return node, node, nil
} }
@ -186,7 +196,7 @@ func (dir *Dir) Lookup(ctx context.Context, name string) (node fs.Node, err erro
if entry.IsDirectory { if entry.IsDirectory {
node = &Dir{Path: path.Join(dir.Path, name), wfs: dir.wfs} node = &Dir{Path: path.Join(dir.Path, name), wfs: dir.wfs}
} else { } else {
node = &File{Chunks: entry.Chunks, Name: name, dir: dir, wfs: dir.wfs}
node = dir.newFile(name, entry.Chunks)
} }
dir.NodeMap[name] = node dir.NodeMap[name] = node
return node, nil return node, nil

113
weed/filesys/file.go

@ -17,7 +17,7 @@ import (
) )
var _ = fs.Node(&File{}) var _ = fs.Node(&File{})
// var _ = fs.NodeOpener(&File{})
var _ = fs.NodeOpener(&File{})
var _ = fs.NodeFsyncer(&File{}) var _ = fs.NodeFsyncer(&File{})
var _ = fs.Handle(&File{}) var _ = fs.Handle(&File{})
var _ = fs.HandleReadAller(&File{}) var _ = fs.HandleReadAller(&File{})
@ -28,58 +28,63 @@ var _ = fs.HandleReleaser(&File{})
var _ = fs.NodeSetattrer(&File{}) var _ = fs.NodeSetattrer(&File{})
type File struct { type File struct {
Chunks []*filer_pb.FileChunk
Name string
dir *Dir
wfs *WFS
Chunks []*filer_pb.FileChunk
Name string
dir *Dir
wfs *WFS
isOpened bool
attributes *filer_pb.FuseAttributes
} }
func (file *File) Attr(context context.Context, attr *fuse.Attr) error { func (file *File) Attr(context context.Context, attr *fuse.Attr) error {
fullPath := filepath.Join(file.dir.Path, file.Name)
item := file.wfs.listDirectoryEntriesCache.Get(fullPath)
var attributes *filer_pb.FuseAttributes
if item != nil {
attributes = item.Value().(*filer_pb.FuseAttributes)
glog.V(1).Infof("read cached file %v attributes", file.Name)
} else {
err := file.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
request := &filer_pb.GetEntryAttributesRequest{
Name: file.Name,
ParentDir: file.dir.Path,
}
glog.V(1).Infof("read file size: %v", request)
resp, err := client.GetEntryAttributes(context, request)
if err != nil {
glog.V(0).Infof("read file attributes %v: %v", request, err)
return err
}
if !file.isOpened || file.attributes == nil {
fullPath := filepath.Join(file.dir.Path, file.Name)
item := file.wfs.listDirectoryEntriesCache.Get(fullPath)
if item != nil {
file.attributes = item.Value().(*filer_pb.FuseAttributes)
glog.V(1).Infof("read cached file %v attributes", file.Name)
} else {
err := file.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
attributes = resp.Attributes
request := &filer_pb.GetEntryAttributesRequest{
Name: file.Name,
ParentDir: file.dir.Path,
}
return nil
})
glog.V(1).Infof("read file size: %v", request)
resp, err := client.GetEntryAttributes(context, request)
if err != nil {
glog.V(0).Infof("read file attributes %v: %v", request, err)
return err
}
if err != nil {
return err
file.attributes = resp.Attributes
return nil
})
if err != nil {
return err
}
} }
} }
attr.Mode = os.FileMode(attributes.FileMode)
attr.Size = attributes.FileSize
attr.Mtime = time.Unix(attributes.Mtime, 0)
attr.Gid = attributes.Gid
attr.Uid = attributes.Uid
attr.Mode = os.FileMode(file.attributes.FileMode)
attr.Size = filer2.TotalSize(file.Chunks)
attr.Mtime = time.Unix(file.attributes.Mtime, 0)
attr.Gid = file.attributes.Gid
attr.Uid = file.attributes.Uid
return nil return nil
} }
func (file *File) xOpen(ctx context.Context, req *fuse.OpenRequest, resp *fuse.OpenResponse) (fs.Handle, error) {
func (file *File) Open(ctx context.Context, req *fuse.OpenRequest, resp *fuse.OpenResponse) (fs.Handle, error) {
fullPath := filepath.Join(file.dir.Path, file.Name) fullPath := filepath.Join(file.dir.Path, file.Name)
fmt.Printf("Open %v %+v\n", fullPath, req) fmt.Printf("Open %v %+v\n", fullPath, req)
file.isOpened = true
return file, nil return file, nil
@ -89,10 +94,28 @@ func (file *File) Setattr(ctx context.Context, req *fuse.SetattrRequest, resp *f
fullPath := filepath.Join(file.dir.Path, file.Name) fullPath := filepath.Join(file.dir.Path, file.Name)
fmt.Printf("Setattr %v %+v\n", fullPath, req) fmt.Printf("Setattr %v %+v\n", fullPath, req)
if req.Valid.Size() && req.Size == 0 {
fmt.Printf("truncate %v \n", fullPath)
file.Chunks = nil
resp.Attr.Size = 0
if req.Valid.Size() {
if req.Size == 0 {
fmt.Printf("truncate %v \n", fullPath)
file.Chunks = nil
}
file.attributes.FileSize = req.Size
}
if req.Valid.Mode() {
file.attributes.FileMode = uint32(req.Mode)
}
if req.Valid.Uid() {
file.attributes.Uid = req.Uid
}
if req.Valid.Gid() {
file.attributes.Gid = req.Gid
}
if req.Valid.Mtime() {
file.attributes.Mtime = req.Mtime.Unix()
} }
return nil return nil
@ -152,16 +175,17 @@ func (file *File) Flush(ctx context.Context, req *fuse.FlushRequest) error {
err := file.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error { err := file.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {
request := &filer_pb.SetFileChunksRequest{
request := &filer_pb.UpdateEntryRequest{
Directory: file.dir.Path, Directory: file.dir.Path,
Entry: &filer_pb.Entry{ Entry: &filer_pb.Entry{
Name: file.Name,
Chunks: file.Chunks,
Name: file.Name,
Attributes: file.attributes,
Chunks: file.Chunks,
}, },
} }
glog.V(1).Infof("append chunks: %v", request) glog.V(1).Infof("append chunks: %v", request)
if _, err := client.SetFileChunks(ctx, request); err != nil {
if _, err := client.UpdateEntry(ctx, request); err != nil {
return fmt.Errorf("create file: %v", err) return fmt.Errorf("create file: %v", err)
} }
@ -224,7 +248,8 @@ func (file *File) Write(ctx context.Context, req *fuse.WriteRequest, resp *fuse.
func (file *File) Release(ctx context.Context, req *fuse.ReleaseRequest) error { func (file *File) Release(ctx context.Context, req *fuse.ReleaseRequest) error {
// fmt.Printf("release file %+v\n", req)
fmt.Printf("release file %+v\n", req)
file.isOpened = false
return nil return nil
} }

6
weed/pb/filer.proto

@ -21,7 +21,7 @@ service SeaweedFiler {
rpc CreateEntry (CreateEntryRequest) returns (CreateEntryResponse) { rpc CreateEntry (CreateEntryRequest) returns (CreateEntryResponse) {
} }
rpc SetFileChunks (SetFileChunksRequest) returns (SetFileChunksResponse) {
rpc UpdateEntry (UpdateEntryRequest) returns (UpdateEntryResponse) {
} }
rpc DeleteEntry (DeleteEntryRequest) returns (DeleteEntryResponse) { rpc DeleteEntry (DeleteEntryRequest) returns (DeleteEntryResponse) {
@ -121,9 +121,9 @@ message AssignVolumeResponse {
int32 count = 4; int32 count = 4;
} }
message SetFileChunksRequest {
message UpdateEntryRequest {
string directory = 1; string directory = 1;
Entry entry = 2; Entry entry = 2;
} }
message SetFileChunksResponse {
message UpdateEntryResponse {
} }

154
weed/pb/filer_pb/filer.pb.go

@ -26,8 +26,8 @@ It has these top-level messages:
DeleteEntryResponse DeleteEntryResponse
AssignVolumeRequest AssignVolumeRequest
AssignVolumeResponse AssignVolumeResponse
SetFileChunksRequest
SetFileChunksResponse
UpdateEntryRequest
UpdateEntryResponse
*/ */
package filer_pb package filer_pb
@ -475,37 +475,37 @@ func (m *AssignVolumeResponse) GetCount() int32 {
return 0 return 0
} }
type SetFileChunksRequest struct {
type UpdateEntryRequest struct {
Directory string `protobuf:"bytes,1,opt,name=directory" json:"directory,omitempty"` Directory string `protobuf:"bytes,1,opt,name=directory" json:"directory,omitempty"`
Entry *Entry `protobuf:"bytes,2,opt,name=entry" json:"entry,omitempty"` Entry *Entry `protobuf:"bytes,2,opt,name=entry" json:"entry,omitempty"`
} }
func (m *SetFileChunksRequest) Reset() { *m = SetFileChunksRequest{} }
func (m *SetFileChunksRequest) String() string { return proto.CompactTextString(m) }
func (*SetFileChunksRequest) ProtoMessage() {}
func (*SetFileChunksRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{17} }
func (m *UpdateEntryRequest) Reset() { *m = UpdateEntryRequest{} }
func (m *UpdateEntryRequest) String() string { return proto.CompactTextString(m) }
func (*UpdateEntryRequest) ProtoMessage() {}
func (*UpdateEntryRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{17} }
func (m *SetFileChunksRequest) GetDirectory() string {
func (m *UpdateEntryRequest) GetDirectory() string {
if m != nil { if m != nil {
return m.Directory return m.Directory
} }
return "" return ""
} }
func (m *SetFileChunksRequest) GetEntry() *Entry {
func (m *UpdateEntryRequest) GetEntry() *Entry {
if m != nil { if m != nil {
return m.Entry return m.Entry
} }
return nil return nil
} }
type SetFileChunksResponse struct {
type UpdateEntryResponse struct {
} }
func (m *SetFileChunksResponse) Reset() { *m = SetFileChunksResponse{} }
func (m *SetFileChunksResponse) String() string { return proto.CompactTextString(m) }
func (*SetFileChunksResponse) ProtoMessage() {}
func (*SetFileChunksResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{18} }
func (m *UpdateEntryResponse) Reset() { *m = UpdateEntryResponse{} }
func (m *UpdateEntryResponse) String() string { return proto.CompactTextString(m) }
func (*UpdateEntryResponse) ProtoMessage() {}
func (*UpdateEntryResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{18} }
func init() { func init() {
proto.RegisterType((*LookupDirectoryEntryRequest)(nil), "filer_pb.LookupDirectoryEntryRequest") proto.RegisterType((*LookupDirectoryEntryRequest)(nil), "filer_pb.LookupDirectoryEntryRequest")
@ -525,8 +525,8 @@ func init() {
proto.RegisterType((*DeleteEntryResponse)(nil), "filer_pb.DeleteEntryResponse") proto.RegisterType((*DeleteEntryResponse)(nil), "filer_pb.DeleteEntryResponse")
proto.RegisterType((*AssignVolumeRequest)(nil), "filer_pb.AssignVolumeRequest") proto.RegisterType((*AssignVolumeRequest)(nil), "filer_pb.AssignVolumeRequest")
proto.RegisterType((*AssignVolumeResponse)(nil), "filer_pb.AssignVolumeResponse") proto.RegisterType((*AssignVolumeResponse)(nil), "filer_pb.AssignVolumeResponse")
proto.RegisterType((*SetFileChunksRequest)(nil), "filer_pb.SetFileChunksRequest")
proto.RegisterType((*SetFileChunksResponse)(nil), "filer_pb.SetFileChunksResponse")
proto.RegisterType((*UpdateEntryRequest)(nil), "filer_pb.UpdateEntryRequest")
proto.RegisterType((*UpdateEntryResponse)(nil), "filer_pb.UpdateEntryResponse")
} }
// Reference imports to suppress errors if they are not otherwise used. // Reference imports to suppress errors if they are not otherwise used.
@ -545,7 +545,7 @@ type SeaweedFilerClient interface {
GetEntryAttributes(ctx context.Context, in *GetEntryAttributesRequest, opts ...grpc.CallOption) (*GetEntryAttributesResponse, error) GetEntryAttributes(ctx context.Context, in *GetEntryAttributesRequest, opts ...grpc.CallOption) (*GetEntryAttributesResponse, error)
GetFileContent(ctx context.Context, in *GetFileContentRequest, opts ...grpc.CallOption) (*GetFileContentResponse, error) GetFileContent(ctx context.Context, in *GetFileContentRequest, opts ...grpc.CallOption) (*GetFileContentResponse, error)
CreateEntry(ctx context.Context, in *CreateEntryRequest, opts ...grpc.CallOption) (*CreateEntryResponse, error) CreateEntry(ctx context.Context, in *CreateEntryRequest, opts ...grpc.CallOption) (*CreateEntryResponse, error)
SetFileChunks(ctx context.Context, in *SetFileChunksRequest, opts ...grpc.CallOption) (*SetFileChunksResponse, error)
UpdateEntry(ctx context.Context, in *UpdateEntryRequest, opts ...grpc.CallOption) (*UpdateEntryResponse, error)
DeleteEntry(ctx context.Context, in *DeleteEntryRequest, opts ...grpc.CallOption) (*DeleteEntryResponse, error) DeleteEntry(ctx context.Context, in *DeleteEntryRequest, opts ...grpc.CallOption) (*DeleteEntryResponse, error)
AssignVolume(ctx context.Context, in *AssignVolumeRequest, opts ...grpc.CallOption) (*AssignVolumeResponse, error) AssignVolume(ctx context.Context, in *AssignVolumeRequest, opts ...grpc.CallOption) (*AssignVolumeResponse, error)
} }
@ -603,9 +603,9 @@ func (c *seaweedFilerClient) CreateEntry(ctx context.Context, in *CreateEntryReq
return out, nil return out, nil
} }
func (c *seaweedFilerClient) SetFileChunks(ctx context.Context, in *SetFileChunksRequest, opts ...grpc.CallOption) (*SetFileChunksResponse, error) {
out := new(SetFileChunksResponse)
err := grpc.Invoke(ctx, "/filer_pb.SeaweedFiler/SetFileChunks", in, out, c.cc, opts...)
func (c *seaweedFilerClient) UpdateEntry(ctx context.Context, in *UpdateEntryRequest, opts ...grpc.CallOption) (*UpdateEntryResponse, error) {
out := new(UpdateEntryResponse)
err := grpc.Invoke(ctx, "/filer_pb.SeaweedFiler/UpdateEntry", in, out, c.cc, opts...)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -638,7 +638,7 @@ type SeaweedFilerServer interface {
GetEntryAttributes(context.Context, *GetEntryAttributesRequest) (*GetEntryAttributesResponse, error) GetEntryAttributes(context.Context, *GetEntryAttributesRequest) (*GetEntryAttributesResponse, error)
GetFileContent(context.Context, *GetFileContentRequest) (*GetFileContentResponse, error) GetFileContent(context.Context, *GetFileContentRequest) (*GetFileContentResponse, error)
CreateEntry(context.Context, *CreateEntryRequest) (*CreateEntryResponse, error) CreateEntry(context.Context, *CreateEntryRequest) (*CreateEntryResponse, error)
SetFileChunks(context.Context, *SetFileChunksRequest) (*SetFileChunksResponse, error)
UpdateEntry(context.Context, *UpdateEntryRequest) (*UpdateEntryResponse, error)
DeleteEntry(context.Context, *DeleteEntryRequest) (*DeleteEntryResponse, error) DeleteEntry(context.Context, *DeleteEntryRequest) (*DeleteEntryResponse, error)
AssignVolume(context.Context, *AssignVolumeRequest) (*AssignVolumeResponse, error) AssignVolume(context.Context, *AssignVolumeRequest) (*AssignVolumeResponse, error)
} }
@ -737,20 +737,20 @@ func _SeaweedFiler_CreateEntry_Handler(srv interface{}, ctx context.Context, dec
return interceptor(ctx, in, info, handler) return interceptor(ctx, in, info, handler)
} }
func _SeaweedFiler_SetFileChunks_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(SetFileChunksRequest)
func _SeaweedFiler_UpdateEntry_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(UpdateEntryRequest)
if err := dec(in); err != nil { if err := dec(in); err != nil {
return nil, err return nil, err
} }
if interceptor == nil { if interceptor == nil {
return srv.(SeaweedFilerServer).SetFileChunks(ctx, in)
return srv.(SeaweedFilerServer).UpdateEntry(ctx, in)
} }
info := &grpc.UnaryServerInfo{ info := &grpc.UnaryServerInfo{
Server: srv, Server: srv,
FullMethod: "/filer_pb.SeaweedFiler/SetFileChunks",
FullMethod: "/filer_pb.SeaweedFiler/UpdateEntry",
} }
handler := func(ctx context.Context, req interface{}) (interface{}, error) { handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(SeaweedFilerServer).SetFileChunks(ctx, req.(*SetFileChunksRequest))
return srv.(SeaweedFilerServer).UpdateEntry(ctx, req.(*UpdateEntryRequest))
} }
return interceptor(ctx, in, info, handler) return interceptor(ctx, in, info, handler)
} }
@ -816,8 +816,8 @@ var _SeaweedFiler_serviceDesc = grpc.ServiceDesc{
Handler: _SeaweedFiler_CreateEntry_Handler, Handler: _SeaweedFiler_CreateEntry_Handler,
}, },
{ {
MethodName: "SetFileChunks",
Handler: _SeaweedFiler_SetFileChunks_Handler,
MethodName: "UpdateEntry",
Handler: _SeaweedFiler_UpdateEntry_Handler,
}, },
{ {
MethodName: "DeleteEntry", MethodName: "DeleteEntry",
@ -835,53 +835,53 @@ var _SeaweedFiler_serviceDesc = grpc.ServiceDesc{
func init() { proto.RegisterFile("filer.proto", fileDescriptor0) } func init() { proto.RegisterFile("filer.proto", fileDescriptor0) }
var fileDescriptor0 = []byte{ var fileDescriptor0 = []byte{
// 765 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xac, 0x56, 0xdd, 0x4e, 0xdb, 0x4a,
0x10, 0xc6, 0x38, 0x09, 0x64, 0x12, 0x38, 0x47, 0x93, 0x00, 0x3e, 0xe1, 0x2f, 0xc7, 0x2d, 0x15,
0x55, 0x25, 0x54, 0xa5, 0x37, 0xbd, 0x2c, 0x02, 0x8a, 0x2a, 0x51, 0x21, 0x19, 0x81, 0x54, 0xf5,
0x22, 0x4a, 0xec, 0x49, 0xba, 0xc2, 0xb1, 0x53, 0x7b, 0xdd, 0x8a, 0x5e, 0xf7, 0x55, 0xfa, 0x1e,
0x7d, 0xb4, 0x6a, 0xd7, 0x1b, 0x7b, 0x4d, 0x1c, 0x5a, 0xa4, 0xde, 0x79, 0xe7, 0xef, 0xfb, 0x76,
0x66, 0xbe, 0x4d, 0xa0, 0x31, 0x62, 0x3e, 0x45, 0x47, 0xd3, 0x28, 0xe4, 0x21, 0xae, 0xca, 0x43,
0x7f, 0x3a, 0xb4, 0x2f, 0x61, 0xfb, 0x22, 0x0c, 0x6f, 0x93, 0xe9, 0x29, 0x8b, 0xc8, 0xe5, 0x61,
0x74, 0x77, 0x16, 0xf0, 0xe8, 0xce, 0xa1, 0xcf, 0x09, 0xc5, 0x1c, 0x77, 0xa0, 0xee, 0xcd, 0x1c,
0x96, 0xd1, 0x35, 0x0e, 0xeb, 0x4e, 0x6e, 0x40, 0x84, 0x4a, 0x30, 0x98, 0x90, 0xb5, 0x2c, 0x1d,
0xf2, 0xdb, 0x3e, 0x83, 0x9d, 0xf2, 0x82, 0xf1, 0x34, 0x0c, 0x62, 0xc2, 0x03, 0xa8, 0x92, 0x30,
0xc8, 0x6a, 0x8d, 0xde, 0x3f, 0x47, 0x33, 0x2a, 0x47, 0x69, 0x5c, 0xea, 0xb5, 0x7b, 0x80, 0x17,
0x2c, 0xe6, 0xc2, 0xc6, 0x28, 0xfe, 0x23, 0x3a, 0xf6, 0x1b, 0x68, 0x15, 0x72, 0x14, 0xe2, 0x73,
0x58, 0xa1, 0xd4, 0x64, 0x19, 0x5d, 0xb3, 0x0c, 0x73, 0xe6, 0xb7, 0x7f, 0x18, 0x50, 0x95, 0xa6,
0xec, 0x6a, 0x46, 0x7e, 0x35, 0xfc, 0x1f, 0x9a, 0x2c, 0xee, 0xe7, 0x04, 0xc4, 0xb5, 0x57, 0x9d,
0x06, 0x8b, 0xb3, 0xab, 0xe2, 0x0b, 0xa8, 0xb9, 0x9f, 0x92, 0xe0, 0x36, 0xb6, 0x4c, 0x09, 0xd5,
0xca, 0xa1, 0xde, 0x32, 0x9f, 0x4e, 0x84, 0xcf, 0x51, 0x21, 0xf8, 0x1a, 0x60, 0xc0, 0x79, 0xc4,
0x86, 0x09, 0xa7, 0xd8, 0xaa, 0xc8, 0x7e, 0x58, 0x5a, 0x42, 0x12, 0xd3, 0x71, 0xe6, 0x77, 0xb4,
0x58, 0x7b, 0x04, 0xf5, 0xac, 0x1c, 0x6e, 0xc1, 0x8a, 0xc8, 0xe9, 0x33, 0x4f, 0xb1, 0xad, 0x89,
0xe3, 0x3b, 0x0f, 0x37, 0xa1, 0x16, 0x8e, 0x46, 0x31, 0x71, 0xc9, 0xd4, 0x74, 0xd4, 0x49, 0xdc,
0x2d, 0x66, 0xdf, 0xc8, 0x32, 0xbb, 0xc6, 0x61, 0xc5, 0x91, 0xdf, 0xd8, 0x86, 0xea, 0x84, 0xb3,
0x09, 0x49, 0x1a, 0xa6, 0x93, 0x1e, 0xec, 0xef, 0x06, 0xac, 0x17, 0x69, 0xe0, 0x36, 0xd4, 0x25,
0x9a, 0xac, 0x60, 0xc8, 0x0a, 0x72, 0x9b, 0xae, 0x0a, 0x55, 0x96, 0xb5, 0x2a, 0x59, 0xca, 0x24,
0xf4, 0x52, 0xd0, 0xb5, 0x34, 0xe5, 0x7d, 0xe8, 0x11, 0xfe, 0x0b, 0x66, 0xc2, 0x3c, 0x09, 0xbb,
0xe6, 0x88, 0x4f, 0x61, 0x19, 0x33, 0xcf, 0xaa, 0xa6, 0x96, 0x31, 0xf3, 0xec, 0x31, 0xfc, 0x77,
0x4e, 0x72, 0xae, 0x77, 0x5a, 0x43, 0xd4, 0x4e, 0x94, 0x4d, 0x6a, 0x17, 0x60, 0x3a, 0x88, 0x28,
0xe0, 0x62, 0x5a, 0x6a, 0x3d, 0xeb, 0xa9, 0xe5, 0x94, 0x45, 0x7a, 0xc7, 0x4c, 0xbd, 0x63, 0xf6,
0x0d, 0x74, 0xca, 0x80, 0xd4, 0x22, 0x15, 0xe7, 0x65, 0x3c, 0x62, 0x5e, 0x2f, 0x61, 0xe3, 0x9c,
0xb8, 0x1c, 0x59, 0x18, 0x70, 0x0a, 0xf8, 0x8c, 0xfc, 0xa2, 0xd9, 0xd9, 0x3d, 0xd8, 0xbc, 0x9f,
0xa1, 0x58, 0x58, 0xb0, 0xe2, 0xa6, 0x26, 0x99, 0xd2, 0x74, 0x66, 0x47, 0xfb, 0x03, 0xe0, 0x49,
0x44, 0x03, 0x4e, 0x8f, 0x90, 0x70, 0x26, 0xc7, 0xe5, 0x07, 0xe5, 0xb8, 0x01, 0xad, 0x42, 0xe9,
0x94, 0x8b, 0xcd, 0x00, 0x4f, 0xc9, 0xa7, 0x47, 0x21, 0x96, 0x3c, 0x1a, 0x73, 0xca, 0x32, 0xe7,
0x94, 0x25, 0x18, 0x14, 0xa0, 0x14, 0x83, 0x09, 0xb4, 0x8e, 0xe3, 0x98, 0x8d, 0x83, 0x9b, 0xd0,
0x4f, 0x26, 0x34, 0xa3, 0xd0, 0x86, 0xaa, 0x1b, 0x26, 0xaa, 0x45, 0x55, 0x27, 0x3d, 0xe0, 0x1e,
0x80, 0x1b, 0xfa, 0x3e, 0xb9, 0x9c, 0x85, 0x81, 0x22, 0xa0, 0x59, 0xb0, 0x0b, 0x8d, 0x88, 0xa6,
0x3e, 0x73, 0x07, 0x32, 0x20, 0xdd, 0x0d, 0xdd, 0x64, 0x7f, 0x81, 0x76, 0x11, 0x4e, 0x0d, 0x65,
0xa1, 0x06, 0xc5, 0x7a, 0x47, 0xbe, 0xc2, 0x12, 0x9f, 0x72, 0x37, 0x93, 0xa1, 0xcf, 0xdc, 0xbe,
0x70, 0x98, 0x6a, 0x37, 0xa5, 0xe5, 0x3a, 0xf2, 0x73, 0xe6, 0x15, 0x8d, 0xb9, 0xfd, 0x11, 0xda,
0x57, 0x6a, 0x1d, 0xe4, 0xdb, 0xf1, 0x57, 0x87, 0xbb, 0x05, 0x1b, 0xf7, 0x8a, 0xa7, 0xb7, 0xea,
0xfd, 0xac, 0x42, 0xf3, 0x8a, 0x06, 0x5f, 0x89, 0x3c, 0xe1, 0x8d, 0x70, 0x0c, 0xed, 0xb2, 0xc7,
0x1d, 0x0f, 0xf2, 0xca, 0x0f, 0xfc, 0x9a, 0x74, 0x9e, 0xfd, 0x2e, 0x4c, 0x0d, 0x75, 0x09, 0x2f,
0xa0, 0xa1, 0x3d, 0xe5, 0xb8, 0xa3, 0x25, 0xce, 0xfd, 0x2a, 0x74, 0x76, 0x17, 0x78, 0xb3, 0x6a,
0x03, 0xc0, 0x79, 0x59, 0xe3, 0x93, 0x3c, 0x6d, 0xe1, 0xeb, 0xd2, 0x79, 0xfa, 0x70, 0x50, 0x06,
0x71, 0x0d, 0xeb, 0x45, 0xbd, 0xe2, 0x7e, 0x21, 0x73, 0x5e, 0xfb, 0x9d, 0xee, 0xe2, 0x00, 0xbd,
0x0f, 0x9a, 0xee, 0xf4, 0x3e, 0xcc, 0x2b, 0x5d, 0xef, 0x43, 0x99, 0x58, 0x97, 0xd0, 0x81, 0xb5,
0xc2, 0xa0, 0x71, 0x2f, 0xcf, 0x28, 0x5b, 0xaf, 0xce, 0xfe, 0x42, 0xbf, 0xce, 0x50, 0xd3, 0xa5,
0xce, 0x70, 0xfe, 0x65, 0xd0, 0x19, 0x96, 0x89, 0x79, 0x09, 0x2f, 0xa1, 0xa9, 0xeb, 0x0b, 0xb5,
0x84, 0x12, 0x99, 0x77, 0xf6, 0x16, 0xb9, 0x67, 0x05, 0x87, 0x35, 0xf9, 0x87, 0xe7, 0xd5, 0xaf,
0x00, 0x00, 0x00, 0xff, 0xff, 0xc7, 0x51, 0xe7, 0x2b, 0xff, 0x08, 0x00, 0x00,
// 754 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xb4, 0x56, 0xdd, 0x6e, 0xd3, 0x4a,
0x10, 0xae, 0xeb, 0x24, 0x6d, 0x26, 0x69, 0xcf, 0xd1, 0x26, 0xed, 0xf1, 0x49, 0x7f, 0x08, 0x86,
0xa2, 0x22, 0xa4, 0x0a, 0x85, 0x1b, 0x2e, 0xa9, 0xda, 0x52, 0x21, 0x15, 0x55, 0x72, 0xd5, 0x4a,
0x5c, 0x45, 0x89, 0x3d, 0x09, 0xab, 0x3a, 0x76, 0xf0, 0xae, 0x41, 0xe5, 0x9a, 0x57, 0xe1, 0x25,
0x78, 0x3a, 0xb4, 0x3f, 0xb1, 0xd7, 0xd8, 0x29, 0xf4, 0x82, 0x3b, 0xef, 0xfc, 0x7c, 0xf3, 0xed,
0xcc, 0x7c, 0x9b, 0x40, 0x6b, 0x42, 0x43, 0x4c, 0x8e, 0xe6, 0x49, 0xcc, 0x63, 0xb2, 0x2e, 0x0f,
0xc3, 0xf9, 0xd8, 0xbd, 0x84, 0x9d, 0x8b, 0x38, 0xbe, 0x4d, 0xe7, 0xa7, 0x34, 0x41, 0x9f, 0xc7,
0xc9, 0xdd, 0x59, 0xc4, 0x93, 0x3b, 0x0f, 0x3f, 0xa5, 0xc8, 0x38, 0xd9, 0x85, 0x66, 0xb0, 0x70,
0x38, 0x56, 0xdf, 0x3a, 0x6c, 0x7a, 0xb9, 0x81, 0x10, 0xa8, 0x45, 0xa3, 0x19, 0x3a, 0xab, 0xd2,
0x21, 0xbf, 0xdd, 0x33, 0xd8, 0xad, 0x06, 0x64, 0xf3, 0x38, 0x62, 0x48, 0x0e, 0xa0, 0x8e, 0xc2,
0x20, 0xd1, 0x5a, 0x83, 0x7f, 0x8e, 0x16, 0x54, 0x8e, 0x54, 0x9c, 0xf2, 0xba, 0x03, 0x20, 0x17,
0x94, 0x71, 0x61, 0xa3, 0xc8, 0xfe, 0x88, 0x8e, 0xfb, 0x06, 0x3a, 0x85, 0x1c, 0x5d, 0xf1, 0x39,
0xac, 0xa1, 0x32, 0x39, 0x56, 0xdf, 0xae, 0xaa, 0xb9, 0xf0, 0xbb, 0xdf, 0x2d, 0xa8, 0x4b, 0x53,
0x76, 0x35, 0x2b, 0xbf, 0x1a, 0x79, 0x0c, 0x6d, 0xca, 0x86, 0x39, 0x01, 0x71, 0xed, 0x75, 0xaf,
0x45, 0x59, 0x76, 0x55, 0xf2, 0x02, 0x1a, 0xfe, 0xc7, 0x34, 0xba, 0x65, 0x8e, 0x2d, 0x4b, 0x75,
0xf2, 0x52, 0x6f, 0x69, 0x88, 0x27, 0xc2, 0xe7, 0xe9, 0x10, 0xf2, 0x1a, 0x60, 0xc4, 0x79, 0x42,
0xc7, 0x29, 0x47, 0xe6, 0xd4, 0x64, 0x3f, 0x1c, 0x23, 0x21, 0x65, 0x78, 0x9c, 0xf9, 0x3d, 0x23,
0xd6, 0x9d, 0x40, 0x33, 0x83, 0x23, 0xff, 0xc1, 0x9a, 0xc8, 0x19, 0xd2, 0x40, 0xb3, 0x6d, 0x88,
0xe3, 0xbb, 0x80, 0x6c, 0x43, 0x23, 0x9e, 0x4c, 0x18, 0x72, 0xc9, 0xd4, 0xf6, 0xf4, 0x49, 0xdc,
0x8d, 0xd1, 0xaf, 0xe8, 0xd8, 0x7d, 0xeb, 0xb0, 0xe6, 0xc9, 0x6f, 0xd2, 0x85, 0xfa, 0x8c, 0xd3,
0x19, 0x4a, 0x1a, 0xb6, 0xa7, 0x0e, 0xee, 0x37, 0x0b, 0x36, 0x8b, 0x34, 0xc8, 0x0e, 0x34, 0x65,
0x35, 0x89, 0x60, 0x49, 0x04, 0xb9, 0x4d, 0x57, 0x05, 0x94, 0x55, 0x03, 0x25, 0x4b, 0x99, 0xc5,
0x81, 0x2a, 0xba, 0xa1, 0x52, 0xde, 0xc7, 0x01, 0x92, 0x7f, 0xc1, 0x4e, 0x69, 0x20, 0xcb, 0x6e,
0x78, 0xe2, 0x53, 0x58, 0xa6, 0x34, 0x70, 0xea, 0xca, 0x32, 0xa5, 0x81, 0x3b, 0x85, 0xff, 0xcf,
0x51, 0xce, 0xf5, 0xce, 0x68, 0x88, 0xde, 0x89, 0xaa, 0x49, 0xed, 0x01, 0xcc, 0x47, 0x09, 0x46,
0x5c, 0x4c, 0x4b, 0xaf, 0x67, 0x53, 0x59, 0x4e, 0x69, 0x62, 0x76, 0xcc, 0x36, 0x3b, 0xe6, 0xde,
0x40, 0xaf, 0xaa, 0x90, 0x5e, 0xa4, 0xe2, 0xbc, 0xac, 0x07, 0xcc, 0xeb, 0x25, 0x6c, 0x9d, 0x23,
0x97, 0x23, 0x8b, 0x23, 0x8e, 0x11, 0x5f, 0x90, 0x5f, 0x36, 0x3b, 0x77, 0x00, 0xdb, 0xbf, 0x66,
0x68, 0x16, 0x0e, 0xac, 0xf9, 0xca, 0x24, 0x53, 0xda, 0xde, 0xe2, 0xe8, 0x7e, 0x00, 0x72, 0x92,
0xe0, 0x88, 0xe3, 0x03, 0x24, 0x9c, 0xc9, 0x71, 0xf5, 0x5e, 0x39, 0x6e, 0x41, 0xa7, 0x00, 0xad,
0xb8, 0xb8, 0x14, 0xc8, 0x29, 0x86, 0xf8, 0xa0, 0x8a, 0x15, 0x8f, 0x46, 0x49, 0x59, 0x76, 0x49,
0x59, 0x82, 0x41, 0xa1, 0x94, 0x66, 0x30, 0x83, 0xce, 0x31, 0x63, 0x74, 0x1a, 0xdd, 0xc4, 0x61,
0x3a, 0xc3, 0x05, 0x85, 0x2e, 0xd4, 0xfd, 0x38, 0xd5, 0x2d, 0xaa, 0x7b, 0xea, 0x40, 0xf6, 0x01,
0xfc, 0x38, 0x0c, 0xd1, 0xe7, 0x34, 0x8e, 0x34, 0x01, 0xc3, 0x42, 0xfa, 0xd0, 0x4a, 0x70, 0x1e,
0x52, 0x7f, 0x24, 0x03, 0xd4, 0x6e, 0x98, 0x26, 0xf7, 0x33, 0x74, 0x8b, 0xe5, 0xf4, 0x50, 0x96,
0x6a, 0x50, 0xac, 0x77, 0x12, 0xea, 0x5a, 0xe2, 0x53, 0xee, 0x66, 0x3a, 0x0e, 0xa9, 0x3f, 0x14,
0x0e, 0x5b, 0xef, 0xa6, 0xb4, 0x5c, 0x27, 0x61, 0xce, 0xbc, 0x66, 0x30, 0x17, 0xa3, 0xbd, 0x9e,
0x07, 0x7f, 0x6b, 0xb4, 0x05, 0x68, 0x75, 0xa3, 0xc1, 0x8f, 0x3a, 0xb4, 0xaf, 0x70, 0xf4, 0x05,
0x31, 0x10, 0x5b, 0x98, 0x90, 0x29, 0x74, 0xab, 0x1e, 0x76, 0x72, 0x90, 0xe3, 0xde, 0xf3, 0x4b,
0xd2, 0x7b, 0xf6, 0xbb, 0x30, 0x3d, 0xd0, 0x15, 0x72, 0x01, 0x2d, 0xe3, 0x19, 0x27, 0xbb, 0x46,
0x62, 0xe9, 0x17, 0xa1, 0xb7, 0xb7, 0xc4, 0x9b, 0xa1, 0x8d, 0x80, 0x94, 0x25, 0x4d, 0x9e, 0xe4,
0x69, 0x4b, 0x5f, 0x96, 0xde, 0xd3, 0xfb, 0x83, 0xb2, 0x12, 0xd7, 0xb0, 0x59, 0xd4, 0x2a, 0x79,
0x54, 0xc8, 0x2c, 0xeb, 0xbe, 0xd7, 0x5f, 0x1e, 0x60, 0xf6, 0xc1, 0xd0, 0x9c, 0xd9, 0x87, 0xb2,
0xca, 0xcd, 0x3e, 0x54, 0x09, 0x55, 0xa2, 0x19, 0x63, 0x36, 0xd1, 0xca, 0x8b, 0x65, 0xa2, 0x55,
0xec, 0x86, 0x42, 0x33, 0xd4, 0x68, 0xa2, 0x95, 0xdf, 0x03, 0x13, 0xad, 0x4a, 0xc2, 0x2b, 0xe4,
0x12, 0xda, 0xa6, 0xaa, 0x88, 0x91, 0x50, 0x21, 0xee, 0xde, 0xfe, 0x32, 0xf7, 0x02, 0x70, 0xdc,
0x90, 0x7f, 0x73, 0x5e, 0xfd, 0x0c, 0x00, 0x00, 0xff, 0xff, 0x9c, 0xf1, 0x24, 0x43, 0xf5, 0x08,
0x00, 0x00,
} }

34
weed/server/filer_grpc_server.go

@ -120,34 +120,46 @@ func (fs *FilerServer) CreateEntry(ctx context.Context, req *filer_pb.CreateEntr
return &filer_pb.CreateEntryResponse{}, err return &filer_pb.CreateEntryResponse{}, err
} }
func (fs *FilerServer) SetFileChunks(ctx context.Context, req *filer_pb.SetFileChunksRequest) (*filer_pb.SetFileChunksResponse, error) {
func (fs *FilerServer) UpdateEntry(ctx context.Context, req *filer_pb.UpdateEntryRequest) (*filer_pb.UpdateEntryResponse, error) {
fullpath := filepath.Join(req.Directory, req.Entry.Name) fullpath := filepath.Join(req.Directory, req.Entry.Name)
found, entry, err := fs.filer.FindEntry(filer2.FullPath(fullpath)) found, entry, err := fs.filer.FindEntry(filer2.FullPath(fullpath))
if err != nil { if err != nil {
return &filer_pb.SetFileChunksResponse{}, err
return &filer_pb.UpdateEntryResponse{}, err
} }
if !found { if !found {
return &filer_pb.SetFileChunksResponse{}, fmt.Errorf("file not found: %s", fullpath)
return &filer_pb.UpdateEntryResponse{}, fmt.Errorf("file not found: %s", fullpath)
} }
chunks := append(entry.Chunks, req.Entry.Chunks...)
// remove old chunks if not included in the new ones
unusedChunks := filer2.FindUnusedFileChunks(entry.Chunks, req.Entry.Chunks)
chunks, garbages := filer2.CompactFileChunks(chunks)
chunks, garbages := filer2.CompactFileChunks(req.Entry.Chunks)
err = fs.filer.SetFileChunks(
filer2.FullPath(fullpath),
chunks,
)
err = fs.filer.UpdateEntry(&filer2.Entry{
FullPath: filer2.FullPath(filepath.Join(req.Directory, req.Entry.Name)),
Attr: filer2.Attr{
Mtime: time.Unix(req.Entry.Attributes.Mtime, 0),
Crtime: time.Unix(req.Entry.Attributes.Mtime, 0),
Mode: os.FileMode(req.Entry.Attributes.FileMode),
Uid: req.Entry.Attributes.Uid,
Gid: req.Entry.Attributes.Gid,
},
Chunks: chunks,
})
if err == nil { if err == nil {
for _, garbage := range garbages {
for _, garbage := range unusedChunks {
glog.V(0).Infof("deleting %s old chunk: %v, [%d, %d)", fullpath, garbage.FileId, garbage.Offset, garbage.Offset+int64(garbage.Size)) glog.V(0).Infof("deleting %s old chunk: %v, [%d, %d)", fullpath, garbage.FileId, garbage.Offset, garbage.Offset+int64(garbage.Size))
operation.DeleteFile(fs.master, garbage.FileId, fs.jwt(garbage.FileId)) operation.DeleteFile(fs.master, garbage.FileId, fs.jwt(garbage.FileId))
} }
for _, garbage := range garbages {
glog.V(0).Infof("deleting %s garbage chunk: %v, [%d, %d)", fullpath, garbage.FileId, garbage.Offset, garbage.Offset+int64(garbage.Size))
operation.DeleteFile(fs.master, garbage.FileId, fs.jwt(garbage.FileId))
}
} }
return &filer_pb.SetFileChunksResponse{}, err
return &filer_pb.UpdateEntryResponse{}, err
} }
func (fs *FilerServer) DeleteEntry(ctx context.Context, req *filer_pb.DeleteEntryRequest) (resp *filer_pb.DeleteEntryResponse, err error) { func (fs *FilerServer) DeleteEntry(ctx context.Context, req *filer_pb.DeleteEntryRequest) (resp *filer_pb.DeleteEntryResponse, err error) {

Loading…
Cancel
Save