From f01d5616b3dba0779d2adbe361271a8f9626e8ee Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sun, 13 May 2018 14:02:29 -0700 Subject: [PATCH 01/83] add better listing directory entries --- weed/filer2/embedded/embedded_store.go | 2 +- weed/filer2/filer.go | 4 ++-- weed/filer2/filer_structure.go | 2 +- weed/filer2/memdb/memdb_store.go | 23 ++++++++++++++++++++--- weed/filer2/memdb/memdb_store_test.go | 15 +++++++++++---- 5 files changed, 35 insertions(+), 11 deletions(-) diff --git a/weed/filer2/embedded/embedded_store.go b/weed/filer2/embedded/embedded_store.go index a2a45807a..a69190e32 100644 --- a/weed/filer2/embedded/embedded_store.go +++ b/weed/filer2/embedded/embedded_store.go @@ -37,6 +37,6 @@ func (filer *EmbeddedStore) DeleteEntry(fullpath filer2.FullPath) (entry *filer2 return nil, nil } -func (filer *EmbeddedStore) ListDirectoryEntries(fullpath filer2.FullPath) (entries []*filer2.Entry, err error) { +func (filer *EmbeddedStore) ListDirectoryEntries(fullpath filer2.FullPath, startFileName string, inclusive bool, limit int) (entries []*filer2.Entry, err error) { return nil, nil } diff --git a/weed/filer2/filer.go b/weed/filer2/filer.go index 39be69d3a..ad7a3d906 100644 --- a/weed/filer2/filer.go +++ b/weed/filer2/filer.go @@ -117,11 +117,11 @@ func (f *Filer) DeleteEntry(p FullPath) (fileEntry *Entry, err error) { return f.store.DeleteEntry(p) } -func (f *Filer) ListDirectoryEntries(p FullPath) ([]*Entry, error) { +func (f *Filer) ListDirectoryEntries(p FullPath, startFileName string, inclusive bool, limit int) ([]*Entry, error) { if strings.HasSuffix(string(p), "/") { p = p[0:len(p)-1] } - return f.store.ListDirectoryEntries(p) + return f.store.ListDirectoryEntries(p, startFileName, inclusive, limit) } func (f *Filer) cacheGetDirectory(dirpath string) (*Entry) { diff --git a/weed/filer2/filer_structure.go b/weed/filer2/filer_structure.go index c6a3f817d..c31d878cf 100644 --- a/weed/filer2/filer_structure.go +++ b/weed/filer2/filer_structure.go @@ -62,5 +62,5 @@ type FilerStore interface { FindEntry(FullPath) (found bool, entry *Entry, err error) DeleteEntry(FullPath) (fileEntry *Entry, err error) - ListDirectoryEntries(dirPath FullPath) ([]*Entry, error) + ListDirectoryEntries(dirPath FullPath, startFileName string, inclusive bool, limit int) ([]*Entry, error) } diff --git a/weed/filer2/memdb/memdb_store.go b/weed/filer2/memdb/memdb_store.go index 47e9934aa..edddcca96 100644 --- a/weed/filer2/memdb/memdb_store.go +++ b/weed/filer2/memdb/memdb_store.go @@ -60,9 +60,18 @@ func (filer *MemDbStore) DeleteEntry(fullpath filer2.FullPath) (entry *filer2.En return entry, nil } -func (filer *MemDbStore) ListDirectoryEntries(fullpath filer2.FullPath) (entries []*filer2.Entry, err error) { - filer.tree.AscendGreaterOrEqual(Entry{&filer2.Entry{FullPath: fullpath}}, +func (filer *MemDbStore) ListDirectoryEntries(fullpath filer2.FullPath, startFileName string, inclusive bool, limit int) (entries []*filer2.Entry, err error) { + + startFrom := string(fullpath) + if startFileName != "" { + startFrom = startFrom + "/" + startFileName + } + + filer.tree.AscendGreaterOrEqual(Entry{&filer2.Entry{FullPath: filer2.FullPath(startFrom)}}, func(item btree.Item) bool { + if limit <= 0 { + return false + } entry := item.(Entry).Entry // println("checking", entry.FullPath) if entry.FullPath == fullpath { @@ -70,7 +79,14 @@ func (filer *MemDbStore) ListDirectoryEntries(fullpath filer2.FullPath) (entries // println("skipping the folder", entry.FullPath) return true } - dir, _ := entry.FullPath.DirAndName() + dir, name := entry.FullPath.DirAndName() + if name == startFileName { + if inclusive { + limit-- + entries = append(entries, entry) + } + return true + } if !strings.HasPrefix(dir, string(fullpath)) { // println("directory is:", dir, "fullpath:", fullpath) // println("breaking from", entry.FullPath) @@ -83,6 +99,7 @@ func (filer *MemDbStore) ListDirectoryEntries(fullpath filer2.FullPath) (entries } // now process the directory items // println("adding entry", entry.FullPath) + limit-- entries = append(entries, entry) return true }, diff --git a/weed/filer2/memdb/memdb_store_test.go b/weed/filer2/memdb/memdb_store_test.go index 9c7810d4d..3fefbf5cd 100644 --- a/weed/filer2/memdb/memdb_store_test.go +++ b/weed/filer2/memdb/memdb_store_test.go @@ -72,7 +72,7 @@ func TestCreateFileAndList(t *testing.T) { filer.CreateEntry(entry2) // checking the 2 files - entries, err := filer.ListDirectoryEntries(filer2.FullPath("/home/chris/this/is/one/")) + entries, err := filer.ListDirectoryEntries(filer2.FullPath("/home/chris/this/is/one/"), "", false, 100) if err != nil { t.Errorf("list entries: %v", err) @@ -94,8 +94,15 @@ func TestCreateFileAndList(t *testing.T) { return } + // checking the offset + entries, err = filer.ListDirectoryEntries(filer2.FullPath("/home/chris/this/is/one/"), "file1.jpg", false, 100) + if len(entries) != 1 { + t.Errorf("list entries count: %v", len(entries)) + return + } + // checking one upper directory - entries, _ = filer.ListDirectoryEntries(filer2.FullPath("/home/chris/this/is")) + entries, _ = filer.ListDirectoryEntries(filer2.FullPath("/home/chris/this/is"), "", false, 100) if len(entries) != 1 { t.Errorf("list entries count: %v", len(entries)) return @@ -113,7 +120,7 @@ func TestCreateFileAndList(t *testing.T) { filer.CreateEntry(entry3) // checking one upper directory - entries, _ = filer.ListDirectoryEntries(filer2.FullPath("/home/chris/this/is")) + entries, _ = filer.ListDirectoryEntries(filer2.FullPath("/home/chris/this/is"), "", false, 100) if len(entries) != 2 { t.Errorf("list entries count: %v", len(entries)) return @@ -121,7 +128,7 @@ func TestCreateFileAndList(t *testing.T) { // delete file and count filer.DeleteEntry(file3Path) - entries, _ = filer.ListDirectoryEntries(filer2.FullPath("/home/chris/this/is")) + entries, _ = filer.ListDirectoryEntries(filer2.FullPath("/home/chris/this/is"), "", false, 100) if len(entries) != 1 { t.Errorf("list entries count: %v", len(entries)) return From c5cf9bd29046877ed6b173e5d2723f865d06aa66 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sun, 13 May 2018 23:56:16 -0700 Subject: [PATCH 02/83] properly working filer --- weed/command/filer_copy.go | 12 +-- weed/filer2/filer.go | 15 ++- weed/filer2/filer_structure.go | 9 ++ weed/filer2/memdb/memdb_store_test.go | 8 ++ weed/operation/filer/register.go | 4 +- weed/server/filer_grpc_server.go | 49 +++++---- weed/server/filer_server.go | 38 +++---- weed/server/filer_server_handlers_admin.go | 42 ++++---- weed/server/filer_server_handlers_read.go | 69 ++++++------ weed/server/filer_server_handlers_write.go | 118 ++++++++++----------- weed/server/filer_ui/templates.go | 23 ++-- 11 files changed, 202 insertions(+), 185 deletions(-) diff --git a/weed/command/filer_copy.go b/weed/command/filer_copy.go index da7fb43bb..2b286d3d8 100644 --- a/weed/command/filer_copy.go +++ b/weed/command/filer_copy.go @@ -91,14 +91,14 @@ func runCopy(cmd *Command, args []string) bool { func doEachCopy(fileOrDir string, host string, path string) bool { f, err := os.Open(fileOrDir) if err != nil { - fmt.Printf("Failed to open file %s: %v", fileOrDir, err) + fmt.Printf("Failed to open file %s: %v\n", fileOrDir, err) return false } defer f.Close() fi, err := f.Stat() if err != nil { - fmt.Printf("Failed to get stat for file %s: %v", fileOrDir, err) + fmt.Printf("Failed to get stat for file %s: %v\n", fileOrDir, err) return false } @@ -122,22 +122,22 @@ func doEachCopy(fileOrDir string, host string, path string) bool { parts, err := operation.NewFileParts([]string{fileOrDir}) if err != nil { - fmt.Printf("Failed to read file %s: %v", fileOrDir, err) + fmt.Printf("Failed to read file %s: %v\n", fileOrDir, err) } results, err := operation.SubmitFiles(*copy.master, parts, *copy.replication, *copy.collection, "", *copy.ttl, *copy.maxMB, copy.secret) if err != nil { - fmt.Printf("Failed to submit file %s: %v", fileOrDir, err) + fmt.Printf("Failed to submit file %s: %v\n", fileOrDir, err) } if strings.HasSuffix(path, "/") { path = path + fi.Name() } - if err = filer_operation.RegisterFile(host, path, results[0].Fid, copy.secret); err != nil { - fmt.Printf("Failed to register file %s on %s: %v", fileOrDir, host, err) + if err = filer_operation.RegisterFile(host, path, results[0].Fid, parts[0].FileSize, copy.secret); err != nil { + fmt.Printf("Failed to register file %s on %s: %v\n", fileOrDir, host, err) return false } diff --git a/weed/filer2/filer.go b/weed/filer2/filer.go index ad7a3d906..5f76d6fb0 100644 --- a/weed/filer2/filer.go +++ b/weed/filer2/filer.go @@ -114,11 +114,24 @@ func (f *Filer) FindEntry(p FullPath) (found bool, entry *Entry, err error) { } func (f *Filer) DeleteEntry(p FullPath) (fileEntry *Entry, err error) { + found, entry, err := f.FindEntry(p) + if err != nil || !found { + return nil, err + } + if entry.IsDirectory() { + entries, err := f.ListDirectoryEntries(p, "", false, 1) + if err != nil { + return nil, fmt.Errorf("list folder %s: %v", p, err) + } + if len(entries) > 0 { + return nil, fmt.Errorf("folder %s is not empty", p) + } + } return f.store.DeleteEntry(p) } func (f *Filer) ListDirectoryEntries(p FullPath, startFileName string, inclusive bool, limit int) ([]*Entry, error) { - if strings.HasSuffix(string(p), "/") { + if strings.HasSuffix(string(p), "/") && len(p) > 1 { p = p[0:len(p)-1] } return f.store.ListDirectoryEntries(p, startFileName, inclusive, limit) diff --git a/weed/filer2/filer_structure.go b/weed/filer2/filer_structure.go index c31d878cf..1cff96253 100644 --- a/weed/filer2/filer_structure.go +++ b/weed/filer2/filer_structure.go @@ -21,6 +21,11 @@ func (fp FullPath) DirAndName() (string, string) { return dir[:len(dir)-1], name } +func (fp FullPath) Name() (string) { + _, name := filepath.Split(string(fp)) + return name +} + type Attr struct { Mtime time.Time // time of last modification Crtime time.Time // time of creation (OS X only) @@ -29,6 +34,10 @@ type Attr struct { Gid uint32 // group gid } +func (attr Attr) IsDirectory() (bool) { + return attr.Mode & os.ModeDir > 0 +} + type Entry struct { FullPath diff --git a/weed/filer2/memdb/memdb_store_test.go b/weed/filer2/memdb/memdb_store_test.go index 3fefbf5cd..ab93a7665 100644 --- a/weed/filer2/memdb/memdb_store_test.go +++ b/weed/filer2/memdb/memdb_store_test.go @@ -108,6 +108,14 @@ func TestCreateFileAndList(t *testing.T) { return } + // checking root directory + entries, _ = filer.ListDirectoryEntries(filer2.FullPath("/"), "", false, 100) + if len(entries) != 1 { + t.Errorf("list entries count: %v", len(entries)) + return + } + + // add file3 file3Path := filer2.FullPath("/home/chris/this/is/file3.jpg") entry3 := &filer2.Entry{ FullPath: file3Path, diff --git a/weed/operation/filer/register.go b/weed/operation/filer/register.go index d45fd4f35..94e502165 100644 --- a/weed/operation/filer/register.go +++ b/weed/operation/filer/register.go @@ -6,6 +6,7 @@ import ( "github.com/chrislusf/seaweedfs/weed/security" "github.com/chrislusf/seaweedfs/weed/util" + "strconv" ) type SubmitResult struct { @@ -16,13 +17,14 @@ type SubmitResult struct { Error string `json:"error,omitempty"` } -func RegisterFile(filer string, path string, fileId string, secret security.Secret) error { +func RegisterFile(filer string, path string, fileId string, fileSize int64, secret security.Secret) error { // TODO: jwt need to be used _ = security.GenJwt(secret, fileId) values := make(url.Values) values.Add("path", path) values.Add("fileId", fileId) + values.Add("fileSize", strconv.FormatInt(fileSize, 10)) _, err := util.Post("http://"+filer+"/admin/register", values) if err != nil { return fmt.Errorf("Failed to register path %s on filer %s to file id %s : %v", path, filer, fileId, err) diff --git a/weed/server/filer_grpc_server.go b/weed/server/filer_grpc_server.go index b53c8c419..665ce2d9b 100644 --- a/weed/server/filer_grpc_server.go +++ b/weed/server/filer_grpc_server.go @@ -8,11 +8,14 @@ import ( "github.com/chrislusf/seaweedfs/weed/util" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "fmt" + "github.com/chrislusf/seaweedfs/weed/filer2" + "path/filepath" + "github.com/chrislusf/seaweedfs/weed/glog" ) func (fs *FilerServer) LookupDirectoryEntry(ctx context.Context, req *filer_pb.LookupDirectoryEntryRequest) (*filer_pb.LookupDirectoryEntryResponse, error) { - found, fileId, err := fs.filer.LookupDirectoryEntry(req.Directory, req.Name) + found, entry, err := fs.filer.FindEntry(filer2.FullPath(filepath.Join(req.Directory, req.Name))) if err != nil { return nil, err } @@ -20,10 +23,15 @@ func (fs *FilerServer) LookupDirectoryEntry(ctx context.Context, req *filer_pb.L return nil, fmt.Errorf("%s not found under %s", req.Name, req.Directory) } + var fileId string + if !entry.IsDirectory() && len(entry.Chunks) > 0 { + fileId = string(entry.Chunks[0].Fid) + } + return &filer_pb.LookupDirectoryEntryResponse{ Entry: &filer_pb.Entry{ Name: req.Name, - IsDirectory: fileId == "", + IsDirectory: entry.IsDirectory(), FileId: fileId, }, }, nil @@ -31,27 +39,21 @@ func (fs *FilerServer) LookupDirectoryEntry(ctx context.Context, req *filer_pb.L func (fs *FilerServer) ListEntries(ctx context.Context, req *filer_pb.ListEntriesRequest) (*filer_pb.ListEntriesResponse, error) { - directoryNames, err := fs.filer.ListDirectories(req.Directory) - if err != nil { - return nil, err - } - files, err := fs.filer.ListFiles(req.Directory, "", 1000) + entries, err := fs.filer.ListDirectoryEntries(filer2.FullPath(req.Directory), "", false, 1000) if err != nil { return nil, err } resp := &filer_pb.ListEntriesResponse{} - for _, dir := range directoryNames { - resp.Entries = append(resp.Entries, &filer_pb.Entry{ - Name: string(dir), - IsDirectory: true, - }) - } - for _, fileEntry := range files { + for _, entry := range entries { + var fileId string + if !entry.IsDirectory() && len(entry.Chunks) > 0 { + fileId = string(entry.Chunks[0].Fid) + } resp.Entries = append(resp.Entries, &filer_pb.Entry{ - Name: fileEntry.Name, - IsDirectory: false, - FileId: string(fileEntry.Id), + Name: entry.Name(), + IsDirectory: entry.IsDirectory(), + FileId: fileId, }) } @@ -97,12 +99,13 @@ func (fs *FilerServer) GetFileContent(ctx context.Context, req *filer_pb.GetFile } func (fs *FilerServer) DeleteEntry(ctx context.Context, req *filer_pb.DeleteEntryRequest) (resp *filer_pb.DeleteEntryResponse, err error) { - if req.IsDirectory { - err = fs.filer.DeleteDirectory(req.Directory+req.Name, false) - } else { - fid, err := fs.filer.DeleteFile(req.Directory + req.Name) - if err == nil && fid != "" { - err = operation.DeleteFile(fs.getMasterNode(), fid, fs.jwt(fid)) + entry, err := fs.filer.DeleteEntry(filer2.FullPath(filepath.Join(req.Directory, req.Name))) + if err == nil { + for _, chunk := range entry.Chunks { + fid := string(chunk.Fid) + if err = operation.DeleteFile(fs.getMasterNode(), fid, fs.jwt(fid)); err != nil { + glog.V(0).Infof("deleting file %s: %v", fid, err) + } } } return nil, err diff --git a/weed/server/filer_server.go b/weed/server/filer_server.go index feaea78ae..8eeced9cb 100644 --- a/weed/server/filer_server.go +++ b/weed/server/filer_server.go @@ -9,21 +9,18 @@ import ( "sync" "time" - "github.com/chrislusf/seaweedfs/weed/filer" - "github.com/chrislusf/seaweedfs/weed/filer/cassandra_store" - "github.com/chrislusf/seaweedfs/weed/filer/embedded_filer" - "github.com/chrislusf/seaweedfs/weed/filer/flat_namespace" "github.com/chrislusf/seaweedfs/weed/filer/mysql_store" "github.com/chrislusf/seaweedfs/weed/filer/postgres_store" - "github.com/chrislusf/seaweedfs/weed/filer/redis_store" "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/security" "github.com/chrislusf/seaweedfs/weed/storage" "github.com/chrislusf/seaweedfs/weed/util" + "github.com/chrislusf/seaweedfs/weed/filer2" + "github.com/chrislusf/seaweedfs/weed/filer2/memdb" ) type filerConf struct { - MysqlConf []mysql_store.MySqlConf `json:"mysql"` + MysqlConf []mysql_store.MySqlConf `json:"mysql"` mysql_store.ShardingConf PostgresConf *postgres_store.PostgresConf `json:"postgres"` } @@ -52,7 +49,7 @@ type FilerServer struct { redirectOnRead bool disableDirListing bool secret security.Secret - filer filer.Filer + filer *filer2.Filer maxMB int masterNodes *storage.MasterNodes } @@ -86,28 +83,31 @@ func NewFilerServer(defaultMux, readonlyMux *http.ServeMux, ip string, port int, } if setting.MysqlConf != nil && len(setting.MysqlConf) != 0 { - mysql_store := mysql_store.NewMysqlStore(setting.MysqlConf, setting.IsSharding, setting.ShardCount) - fs.filer = flat_namespace.NewFlatNamespaceFiler(master, mysql_store) + // mysql_store := mysql_store.NewMysqlStore(setting.MysqlConf, setting.IsSharding, setting.ShardCount) + // fs.filer = flat_namespace.NewFlatNamespaceFiler(master, mysql_store) } else if setting.PostgresConf != nil { - fs.filer = postgres_store.NewPostgresStore(master, *setting.PostgresConf) + // fs.filer = postgres_store.NewPostgresStore(master, *setting.PostgresConf) } else if cassandra_server != "" { - cassandra_store, err := cassandra_store.NewCassandraStore(cassandra_keyspace, cassandra_server) - if err != nil { - glog.Fatalf("Can not connect to cassandra server %s with keyspace %s: %v", cassandra_server, cassandra_keyspace, err) - } - fs.filer = flat_namespace.NewFlatNamespaceFiler(master, cassandra_store) + // cassandra_store, err := cassandra_store.NewCassandraStore(cassandra_keyspace, cassandra_server) + // if err != nil { + // glog.Fatalf("Can not connect to cassandra server %s with keyspace %s: %v", cassandra_server, cassandra_keyspace, err) + // } + // fs.filer = flat_namespace.NewFlatNamespaceFiler(master, cassandra_store) } else if redis_server != "" { - redis_store := redis_store.NewRedisStore(redis_server, redis_password, redis_database) - fs.filer = flat_namespace.NewFlatNamespaceFiler(master, redis_store) + // redis_store := redis_store.NewRedisStore(redis_server, redis_password, redis_database) + // fs.filer = flat_namespace.NewFlatNamespaceFiler(master, redis_store) } else { + /* if fs.filer, err = embedded_filer.NewFilerEmbedded(master, dir); err != nil { glog.Fatalf("Can not start filer in dir %s : %v", dir, err) return } - - defaultMux.HandleFunc("/admin/mv", fs.moveHandler) + */ } + fs.filer = filer2.NewFiler(master) + fs.filer.SetStore(memdb.NewMemDbStore()) + defaultMux.HandleFunc("/admin/register", fs.registerHandler) defaultMux.HandleFunc("/", fs.filerHandler) if defaultMux != readonlyMux { diff --git a/weed/server/filer_server_handlers_admin.go b/weed/server/filer_server_handlers_admin.go index aa7c09986..5c1b23255 100644 --- a/weed/server/filer_server_handlers_admin.go +++ b/weed/server/filer_server_handlers_admin.go @@ -4,34 +4,30 @@ import ( "net/http" "github.com/chrislusf/seaweedfs/weed/glog" + "github.com/chrislusf/seaweedfs/weed/filer2" + "strconv" ) -/* -Move a folder or a file, with 4 Use cases: - mv fromDir toNewDir - mv fromDir toOldDir - mv fromFile toDir - mv fromFile toFile - -Wildcard is not supported. - -*/ -func (fs *FilerServer) moveHandler(w http.ResponseWriter, r *http.Request) { - from := r.FormValue("from") - to := r.FormValue("to") - err := fs.filer.Move(from, to) - if err != nil { - glog.V(4).Infoln("moving", from, "->", to, err.Error()) - writeJsonError(w, r, http.StatusInternalServerError, err) - } else { - w.WriteHeader(http.StatusOK) - } -} - func (fs *FilerServer) registerHandler(w http.ResponseWriter, r *http.Request) { path := r.FormValue("path") fileId := r.FormValue("fileId") - err := fs.filer.CreateFile(path, fileId) + fileSize, err := strconv.ParseUint(r.FormValue("fileSize"), 10, 64) + if err != nil { + glog.V(4).Infof("register %s to %s parse fileSize %s: %v", fileId, path, r.FormValue("fileSize"), err) + writeJsonError(w, r, http.StatusInternalServerError, err) + return + } + entry := &filer2.Entry{ + FullPath: filer2.FullPath(path), + Attr: filer2.Attr{ + Mode: 0660, + }, + Chunks: []filer2.FileChunk{{ + Fid: filer2.FileId(fileId), + Size: fileSize, + }}, + } + err = fs.filer.CreateEntry(entry) if err != nil { glog.V(4).Infof("register %s to %s error: %v", fileId, path, err) writeJsonError(w, r, http.StatusInternalServerError, err) diff --git a/weed/server/filer_server_handlers_read.go b/weed/server/filer_server_handlers_read.go index e95c7fcd0..93775f839 100644 --- a/weed/server/filer_server_handlers_read.go +++ b/weed/server/filer_server_handlers_read.go @@ -7,12 +7,11 @@ import ( "strconv" "strings" - "github.com/chrislusf/seaweedfs/weed/filer" "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/operation" ui "github.com/chrislusf/seaweedfs/weed/server/filer_ui" "github.com/chrislusf/seaweedfs/weed/util" - "github.com/syndtr/goleveldb/leveldb" + "github.com/chrislusf/seaweedfs/weed/filer2" ) // listDirectoryHandler lists directories and folers under a directory @@ -20,56 +19,40 @@ import ( // sub directories are listed on the first page, when "lastFileName" // is empty. func (fs *FilerServer) listDirectoryHandler(w http.ResponseWriter, r *http.Request) { - if !strings.HasSuffix(r.URL.Path, "/") { - return + path := r.URL.Path + if strings.HasSuffix(path, "/") && len(path) > 1 { + path = path[:len(path)-1] } + limit, limit_err := strconv.Atoi(r.FormValue("limit")) if limit_err != nil { limit = 100 } lastFileName := r.FormValue("lastFileName") - files, err := fs.filer.ListFiles(r.URL.Path, lastFileName, limit) - if err == leveldb.ErrNotFound { - glog.V(0).Infof("Error %s", err) - w.WriteHeader(http.StatusNotFound) - return - } + entries, err := fs.filer.ListDirectoryEntries(filer2.FullPath(path), lastFileName, false, limit) - directories, err2 := fs.filer.ListDirectories(r.URL.Path) - if err2 == leveldb.ErrNotFound { - glog.V(0).Infof("Error %s", err) + if err != nil { + glog.V(0).Infof("listDirectory %s %s $d: %s", path, lastFileName, limit, err) w.WriteHeader(http.StatusNotFound) return } - shouldDisplayLoadMore := len(files) > 0 - - lastFileName = "" - if len(files) > 0 { - lastFileName = files[len(files)-1].Name - - files2, err3 := fs.filer.ListFiles(r.URL.Path, lastFileName, limit) - if err3 == leveldb.ErrNotFound { - glog.V(0).Infof("Error %s", err) - w.WriteHeader(http.StatusNotFound) - return - } - shouldDisplayLoadMore = len(files2) > 0 + shouldDisplayLoadMore := len(entries) == limit + if path == "/" { + path = "" } args := struct { Path string - Files interface{} - Directories interface{} + Entries interface{} Limit int LastFileName string ShouldDisplayLoadMore bool }{ - r.URL.Path, - files, - directories, + path, + entries, limit, lastFileName, shouldDisplayLoadMore, @@ -83,7 +66,19 @@ func (fs *FilerServer) listDirectoryHandler(w http.ResponseWriter, r *http.Reque } func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request, isGetMethod bool) { - if strings.HasSuffix(r.URL.Path, "/") { + path := r.URL.Path + if strings.HasSuffix(path, "/") && len(path) > 1 { + path = path[:len(path)-1] + } + + found, entry, err := fs.filer.FindEntry(filer2.FullPath(path)) + if !found || err != nil { + glog.V(3).Infof("Not found %s: %v", path, err) + w.WriteHeader(http.StatusNotFound) + return + } + + if entry.IsDirectory() { if fs.disableDirListing { w.WriteHeader(http.StatusMethodNotAllowed) return @@ -92,13 +87,15 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request, return } - fileId, err := fs.filer.FindFile(r.URL.Path) - if err == filer.ErrNotFound { - glog.V(3).Infoln("Not found in db", r.URL.Path) - w.WriteHeader(http.StatusNotFound) + if len(entry.Chunks) == 0 { + glog.V(3).Infof("Empty %s: %v", path) + w.WriteHeader(http.StatusNoContent) return } + // FIXME pick the right fid + fileId := string(entry.Chunks[0].Fid) + urlLocation, err := operation.LookupFileId(fs.getMasterNode(), fileId) if err != nil { glog.V(1).Infoln("operation LookupFileId %s failed, err is %s", fileId, err.Error()) diff --git a/weed/server/filer_server_handlers_write.go b/weed/server/filer_server_handlers_write.go index 07452cd77..1a4b62235 100644 --- a/weed/server/filer_server_handlers_write.go +++ b/weed/server/filer_server_handlers_write.go @@ -22,6 +22,7 @@ import ( "github.com/chrislusf/seaweedfs/weed/operation" "github.com/chrislusf/seaweedfs/weed/storage" "github.com/chrislusf/seaweedfs/weed/util" + "github.com/chrislusf/seaweedfs/weed/filer2" ) type FilerPostResult struct { @@ -73,16 +74,19 @@ func makeFormData(filename, mimeType string, content io.Reader) (formData io.Rea } func (fs *FilerServer) queryFileInfoByPath(w http.ResponseWriter, r *http.Request, path string) (fileId, urlLocation string, err error) { - if fileId, err = fs.filer.FindFile(path); err != nil && err != filer.ErrNotFound { + var found bool + var entry *filer2.Entry + if found, entry, err = fs.filer.FindEntry(filer2.FullPath(path)); err != nil { glog.V(0).Infoln("failing to find path in filer store", path, err.Error()) writeJsonError(w, r, http.StatusInternalServerError, err) - } else if fileId != "" && err == nil { + } else if found { + fileId = string(entry.Chunks[0].Fid) urlLocation, err = operation.LookupFileId(fs.getMasterNode(), fileId) if err != nil { glog.V(1).Infoln("operation LookupFileId %s failed, err is %s", fileId, err.Error()) w.WriteHeader(http.StatusNotFound) } - } else if fileId == "" && err == filer.ErrNotFound { + } else { w.WriteHeader(http.StatusNotFound) } return @@ -313,7 +317,8 @@ func (fs *FilerServer) PostHandler(w http.ResponseWriter, r *http.Request) { // also delete the old fid unless PUT operation if r.Method != "PUT" { - if oldFid, err := fs.filer.FindFile(path); err == nil { + if found, entry, err := fs.filer.FindEntry(filer2.FullPath(path)); err == nil && found { + oldFid := string(entry.Chunks[0].Fid) operation.DeleteFile(fs.getMasterNode(), oldFid, fs.jwt(oldFid)) } else if err != nil && err != filer.ErrNotFound { glog.V(0).Infof("error %v occur when finding %s in filer store", err, path) @@ -321,7 +326,17 @@ func (fs *FilerServer) PostHandler(w http.ResponseWriter, r *http.Request) { } glog.V(4).Infoln("saving", path, "=>", fileId) - if db_err := fs.filer.CreateFile(path, fileId); db_err != nil { + entry := &filer2.Entry{ + FullPath: filer2.FullPath(path), + Attr: filer2.Attr{ + Mode: 0660, + }, + Chunks: []filer2.FileChunk{{ + Fid: filer2.FileId(fileId), + Size: uint64(r.ContentLength), + }}, + } + if db_err := fs.filer.CreateEntry(entry); db_err != nil { operation.DeleteFile(fs.getMasterNode(), fileId, fs.jwt(fileId)) //clean up glog.V(0).Infof("failing to write %s to filer server : %v", path, db_err) writeJsonError(w, r, http.StatusInternalServerError, db_err) @@ -400,13 +415,7 @@ func (fs *FilerServer) doAutoChunk(w http.ResponseWriter, r *http.Request, conte fileName = path.Base(fileName) } - chunks := (int64(contentLength) / int64(chunkSize)) + 1 - cm := operation.ChunkManifest{ - Name: fileName, - Size: 0, // don't know yet - Mime: "application/octet-stream", - Chunks: make([]*operation.ChunkInfo, 0, chunks), - } + var fileChunks []filer2.FileChunk totalBytesRead := int64(0) tmpBufferSize := int32(1024 * 1024) @@ -438,18 +447,18 @@ func (fs *FilerServer) doAutoChunk(w http.ResponseWriter, r *http.Request, conte } // upload the chunk to the volume server - chunkName := fileName + "_chunk_" + strconv.FormatInt(int64(cm.Chunks.Len()+1), 10) + chunkName := fileName + "_chunk_" + strconv.FormatInt(int64(len(fileChunks)+1), 10) uploadErr := fs.doUpload(urlLocation, w, r, chunkBuf[0:chunkBufOffset], chunkName, "application/octet-stream", fileId) if uploadErr != nil { return nil, uploadErr } // Save to chunk manifest structure - cm.Chunks = append(cm.Chunks, - &operation.ChunkInfo{ + fileChunks = append(fileChunks, + filer2.FileChunk{ + Fid: filer2.FileId(fileId), Offset: chunkOffset, - Size: int64(chunkBufOffset), - Fid: fileId, + Size: uint64(chunkBufOffset), }, ) @@ -469,47 +478,30 @@ func (fs *FilerServer) doAutoChunk(w http.ResponseWriter, r *http.Request, conte } } - cm.Size = totalBytesRead - manifestBuf, marshalErr := cm.Marshal() - if marshalErr != nil { - return nil, marshalErr - } - - manifestStr := string(manifestBuf) - glog.V(4).Infoln("Generated chunk manifest: ", manifestStr) - - manifestFileId, manifestUrlLocation, manifestAssignmentErr := fs.assignNewFileInfo(w, r, replication, collection) - if manifestAssignmentErr != nil { - return nil, manifestAssignmentErr - } - glog.V(4).Infoln("Manifest uploaded to:", manifestUrlLocation, "Fid:", manifestFileId) - filerResult.Fid = manifestFileId - - u, _ := url.Parse(manifestUrlLocation) - q := u.Query() - q.Set("cm", "true") - u.RawQuery = q.Encode() - - manifestUploadErr := fs.doUpload(u.String(), w, r, manifestBuf, fileName+"_manifest", "application/json", manifestFileId) - if manifestUploadErr != nil { - return nil, manifestUploadErr - } - path := r.URL.Path // also delete the old fid unless PUT operation if r.Method != "PUT" { - if oldFid, err := fs.filer.FindFile(path); err == nil { - operation.DeleteFile(fs.getMasterNode(), oldFid, fs.jwt(oldFid)) - } else if err != nil && err != filer.ErrNotFound { + if found, entry, err := fs.filer.FindEntry(filer2.FullPath(path)); found && err == nil { + for _, chunk := range entry.Chunks { + oldFid := string(chunk.Fid) + operation.DeleteFile(fs.getMasterNode(), oldFid, fs.jwt(oldFid)) + } + } else if err != nil { glog.V(0).Infof("error %v occur when finding %s in filer store", err, path) } } - glog.V(4).Infoln("saving", path, "=>", manifestFileId) - if db_err := fs.filer.CreateFile(path, manifestFileId); db_err != nil { + glog.V(4).Infoln("saving", path) + entry := &filer2.Entry{ + FullPath: filer2.FullPath(path), + Attr: filer2.Attr{ + Mode: 0660, + }, + Chunks: fileChunks, + } + if db_err := fs.filer.CreateEntry(entry); db_err != nil { replyerr = db_err filerResult.Error = db_err.Error() - operation.DeleteFile(fs.getMasterNode(), manifestFileId, fs.jwt(manifestFileId)) //clean up glog.V(0).Infof("failing to write %s to filer server : %v", path, db_err) return } @@ -532,23 +524,21 @@ func (fs *FilerServer) doUpload(urlLocation string, w http.ResponseWriter, r *ht } // curl -X DELETE http://localhost:8888/path/to -// curl -X DELETE http://localhost:8888/path/to/?recursive=true func (fs *FilerServer) DeleteHandler(w http.ResponseWriter, r *http.Request) { - var err error - var fid string - if strings.HasSuffix(r.URL.Path, "/") { - isRecursive := r.FormValue("recursive") == "true" - err = fs.filer.DeleteDirectory(r.URL.Path, isRecursive) - } else { - fid, err = fs.filer.DeleteFile(r.URL.Path) - if err == nil && fid != "" { - err = operation.DeleteFile(fs.getMasterNode(), fid, fs.jwt(fid)) - } - } - if err == nil { - writeJsonQuiet(w, r, http.StatusAccepted, map[string]string{"error": ""}) - } else { + + entry, err := fs.filer.DeleteEntry(filer2.FullPath(r.URL.Path)) + if err != nil { glog.V(4).Infoln("deleting", r.URL.Path, ":", err.Error()) writeJsonError(w, r, http.StatusInternalServerError, err) + return + } + + if entry != nil && !entry.IsDirectory() { + for _, chunk := range entry.Chunks { + oldFid := string(chunk.Fid) + operation.DeleteFile(fs.getMasterNode(), oldFid, fs.jwt(oldFid)) + } } + + writeJsonQuiet(w, r, http.StatusAccepted, map[string]string{"error": ""}) } diff --git a/weed/server/filer_ui/templates.go b/weed/server/filer_ui/templates.go index 508a4d9aa..bba60ff1a 100644 --- a/weed/server/filer_ui/templates.go +++ b/weed/server/filer_ui/templates.go @@ -26,22 +26,21 @@ var StatusTpl = template.Must(template.New("status").Parse(`
From 67401f190851f645e54687c502f9af6ad0e6651d Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Mon, 14 May 2018 00:17:22 -0700 Subject: [PATCH 03/83] fix gRpc nil response --- weed/server/filer_grpc_server.go | 2 +- weed/util/constants.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/weed/server/filer_grpc_server.go b/weed/server/filer_grpc_server.go index 665ce2d9b..e5b30aedf 100644 --- a/weed/server/filer_grpc_server.go +++ b/weed/server/filer_grpc_server.go @@ -108,5 +108,5 @@ func (fs *FilerServer) DeleteEntry(ctx context.Context, req *filer_pb.DeleteEntr } } } - return nil, err + return &filer_pb.DeleteEntryResponse{}, err } diff --git a/weed/util/constants.go b/weed/util/constants.go index 96cc83108..f96bfebca 100644 --- a/weed/util/constants.go +++ b/weed/util/constants.go @@ -1,5 +1,5 @@ package util const ( - VERSION = "0.77" + VERSION = "0.78" ) From 58954bf46f5098c1181920bd04ac9e45f6b97913 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Mon, 14 May 2018 02:02:17 -0700 Subject: [PATCH 04/83] pass file attributes from filer to mount --- weed/filer2/filer_structure.go | 9 +++++- weed/filesys/dir.go | 4 ++- weed/filesys/file.go | 48 +++++++++++++++++++++++--------- weed/filesys/wfs.go | 7 +++-- weed/server/filer_grpc_server.go | 28 ++++++++++++------- 5 files changed, 69 insertions(+), 27 deletions(-) diff --git a/weed/filer2/filer_structure.go b/weed/filer2/filer_structure.go index 1cff96253..b2c1f5902 100644 --- a/weed/filer2/filer_structure.go +++ b/weed/filer2/filer_structure.go @@ -10,6 +10,13 @@ import ( type FileId string //file id in SeaweedFS type FullPath string +func NewFullPath(dir, name string) FullPath { + if dir == "/" { + return FullPath(dir + name) + } + return FullPath(dir + "/" + name) +} + func (fp FullPath) DirAndName() (string, string) { dir, name := filepath.Split(string(fp)) if dir == "/" { @@ -35,7 +42,7 @@ type Attr struct { } func (attr Attr) IsDirectory() (bool) { - return attr.Mode & os.ModeDir > 0 + return attr.Mode&os.ModeDir > 0 } type Entry struct { diff --git a/weed/filesys/dir.go b/weed/filesys/dir.go index c1c6afe9c..d408dc1ac 100644 --- a/weed/filesys/dir.go +++ b/weed/filesys/dir.go @@ -12,6 +12,7 @@ import ( "github.com/chrislusf/seaweedfs/weed/filer" "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" + "time" ) type Dir struct { @@ -74,7 +75,7 @@ func (dir *Dir) Lookup(ctx context.Context, name string) (node fs.Node, err erro if entry.IsDirectory { node = &Dir{Path: path.Join(dir.Path, name), wfs: dir.wfs} } else { - node = &File{FileId: filer.FileId(entry.FileId), Name: name, wfs: dir.wfs} + node = &File{FileId: filer.FileId(entry.FileId), Name: name, dir: dir, wfs: dir.wfs} } dir.NodeMap[name] = node return node, nil @@ -104,6 +105,7 @@ func (dir *Dir) ReadDirAll(ctx context.Context) (ret []fuse.Dirent, err error) { } else { dirent := fuse.Dirent{Name: entry.Name, Type: fuse.DT_File} ret = append(ret, dirent) + dir.wfs.listDirectoryEntriesCache.Set(dir.Path+"/"+entry.Name, entry.Attributes, 3*time.Second) } } diff --git a/weed/filesys/file.go b/weed/filesys/file.go index e4c79c055..b4a447fa6 100644 --- a/weed/filesys/file.go +++ b/weed/filesys/file.go @@ -9,6 +9,9 @@ import ( "bazil.org/fuse/fs" "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" + "path/filepath" + "os" + "time" ) var _ = fs.Node(&File{}) @@ -22,29 +25,48 @@ var _ = fs.HandleWriter(&File{}) type File struct { FileId filer.FileId Name string + dir *Dir wfs *WFS } func (file *File) Attr(context context.Context, attr *fuse.Attr) error { - attr.Mode = 0444 - return file.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error { + fullPath := filepath.Join(file.dir.Path, file.Name) + item := file.wfs.listDirectoryEntriesCache.Get(fullPath) + var attributes *filer_pb.FuseAttributes + if item != nil { + attributes = item.Value().(*filer_pb.FuseAttributes) + glog.V(1).Infof("read cached file %v attributes", file.Name) + } else { + err := file.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error { + + request := &filer_pb.GetFileAttributesRequest{ + Name: file.Name, + ParentDir: file.dir.Path, + } + + glog.V(1).Infof("read file size: %v", request) + resp, err := client.GetFileAttributes(context, request) + if err != nil { + return err + } + + attributes = resp.Attributes + + return nil + }) - request := &filer_pb.GetFileAttributesRequest{ - Name: file.Name, - ParentDir: "", //TODO add parent folder - FileId: string(file.FileId), - } - - glog.V(1).Infof("read file size: %v", request) - resp, err := client.GetFileAttributes(context, request) if err != nil { return err } + } - attr.Size = resp.Attributes.FileSize + attr.Mode = os.FileMode(attributes.FileMode) + attr.Size = attributes.FileSize + attr.Mtime = time.Unix(attributes.Mtime, 0) + attr.Gid = attributes.Gid + attr.Uid = attributes.Uid + return nil - return nil - }) } func (file *File) ReadAll(ctx context.Context) (content []byte, err error) { diff --git a/weed/filesys/wfs.go b/weed/filesys/wfs.go index da50ae4a4..9f8f4649a 100644 --- a/weed/filesys/wfs.go +++ b/weed/filesys/wfs.go @@ -5,15 +5,18 @@ import ( "fmt" "google.golang.org/grpc" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" + "github.com/karlseguin/ccache" ) type WFS struct { - filer string + filer string + listDirectoryEntriesCache *ccache.Cache } func NewSeaweedFileSystem(filer string) *WFS { return &WFS{ - filer: filer, + filer: filer, + listDirectoryEntriesCache: ccache.New(ccache.Configure().MaxSize(6000).ItemsToPrune(100)), } } diff --git a/weed/server/filer_grpc_server.go b/weed/server/filer_grpc_server.go index e5b30aedf..4d31782da 100644 --- a/weed/server/filer_grpc_server.go +++ b/weed/server/filer_grpc_server.go @@ -2,8 +2,6 @@ package weed_server import ( "context" - "strconv" - "github.com/chrislusf/seaweedfs/weed/operation" "github.com/chrislusf/seaweedfs/weed/util" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" @@ -50,10 +48,19 @@ func (fs *FilerServer) ListEntries(ctx context.Context, req *filer_pb.ListEntrie if !entry.IsDirectory() && len(entry.Chunks) > 0 { fileId = string(entry.Chunks[0].Fid) } + + glog.V(0).Infof("%s attr=%v size=%d", entry.Name(), entry.Attr, filer2.Chunks(entry.Chunks).TotalSize()) resp.Entries = append(resp.Entries, &filer_pb.Entry{ Name: entry.Name(), IsDirectory: entry.IsDirectory(), FileId: fileId, + Attributes: &filer_pb.FuseAttributes{ + FileSize: filer2.Chunks(entry.Chunks).TotalSize(), + Mtime: entry.Mtime.Unix(), + Gid: entry.Gid, + Uid: entry.Uid, + FileMode: uint32(entry.Mode), + }, }) } @@ -64,17 +71,18 @@ func (fs *FilerServer) GetFileAttributes(ctx context.Context, req *filer_pb.GetF attributes := &filer_pb.FuseAttributes{} - server, err := operation.LookupFileId(fs.getMasterNode(), req.FileId) + found, entry, err := fs.filer.FindEntry(filer2.NewFullPath(req.ParentDir, req.Name)) if err != nil { return nil, err } - head, err := util.Head(server) - if err != nil { - return nil, err - } - attributes.FileSize, err = strconv.ParseUint(head.Get("Content-Length"), 10, 0) - if err != nil { - return nil, err + if !found { + attributes.FileSize = 0 + } else { + attributes.FileSize = filer2.Chunks(entry.Chunks).TotalSize() + attributes.FileMode = uint32(entry.Mode) + attributes.Uid = entry.Uid + attributes.Gid = entry.Gid + attributes.Mtime = entry.Mtime.Unix() } return &filer_pb.GetFileAttributesResponse{ From c7a71d35b0f4c94361f227396e12412e4333a2ba Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Mon, 14 May 2018 20:27:48 -0700 Subject: [PATCH 05/83] fix on pagination --- weed/server/filer_server_handlers_read.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/weed/server/filer_server_handlers_read.go b/weed/server/filer_server_handlers_read.go index 93775f839..9d13757be 100644 --- a/weed/server/filer_server_handlers_read.go +++ b/weed/server/filer_server_handlers_read.go @@ -44,6 +44,10 @@ func (fs *FilerServer) listDirectoryHandler(w http.ResponseWriter, r *http.Reque path = "" } + if len(entries) > 0 { + lastFileName = entries[len(entries)-1].Name() + } + args := struct { Path string Entries interface{} From b303a02461a009298e27f7b9b4f8ae68c8f44f21 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Wed, 16 May 2018 00:08:44 -0700 Subject: [PATCH 06/83] cp file can work 1. consolidate to filer_pb.FileChunk 2. dir add file, mkdir 3. file flush, write updates having issue --- weed/filer2/embedded/embedded_store.go | 3 +- weed/filer2/filechunks.go | 4 +- weed/filer2/filer.go | 5 +- weed/filer2/filer_structure.go | 14 +- weed/filer2/memdb/memdb_store.go | 7 +- weed/filesys/dir.go | 74 +++- weed/filesys/file.go | 81 ++++- weed/operation/assign_file_id.go | 2 +- weed/pb/filer.proto | 45 ++- weed/pb/filer_pb/filer.pb.go | 387 ++++++++++++++++++--- weed/server/filer_grpc_server.go | 69 +++- weed/server/filer_server_handlers_admin.go | 7 +- weed/server/filer_server_handlers_read.go | 2 +- weed/server/filer_server_handlers_write.go | 21 +- 14 files changed, 619 insertions(+), 102 deletions(-) diff --git a/weed/filer2/embedded/embedded_store.go b/weed/filer2/embedded/embedded_store.go index a69190e32..f871f18e2 100644 --- a/weed/filer2/embedded/embedded_store.go +++ b/weed/filer2/embedded/embedded_store.go @@ -3,6 +3,7 @@ package embedded import ( "github.com/syndtr/goleveldb/leveldb" "github.com/chrislusf/seaweedfs/weed/filer2" + "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" ) type EmbeddedStore struct { @@ -25,7 +26,7 @@ func (filer *EmbeddedStore) AddDirectoryLink(directory *filer2.Entry, delta int3 return nil } -func (filer *EmbeddedStore) AppendFileChunk(fullpath filer2.FullPath, fileChunk filer2.FileChunk) (err error) { +func (filer *EmbeddedStore) AppendFileChunk(fullpath filer2.FullPath, fileChunks []*filer_pb.FileChunk) (err error) { return nil } diff --git a/weed/filer2/filechunks.go b/weed/filer2/filechunks.go index b2f05de3a..9ddbe236e 100644 --- a/weed/filer2/filechunks.go +++ b/weed/filer2/filechunks.go @@ -1,6 +1,8 @@ package filer2 -type Chunks []FileChunk +import "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" + +type Chunks []*filer_pb.FileChunk func (chunks Chunks) TotalSize() (size uint64) { for _, c := range chunks { diff --git a/weed/filer2/filer.go b/weed/filer2/filer.go index 5f76d6fb0..4ee6f4e50 100644 --- a/weed/filer2/filer.go +++ b/weed/filer2/filer.go @@ -8,6 +8,7 @@ import ( "path/filepath" "time" "os" + "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" ) type Filer struct { @@ -105,8 +106,8 @@ func (f *Filer) CreateEntry(entry *Entry) (error) { return nil } -func (f *Filer) AppendFileChunk(p FullPath, c FileChunk) (err error) { - return f.store.AppendFileChunk(p, c) +func (f *Filer) AppendFileChunk(p FullPath, chunks []*filer_pb.FileChunk) (err error) { + return f.store.AppendFileChunk(p, chunks) } func (f *Filer) FindEntry(p FullPath) (found bool, entry *Entry, err error) { diff --git a/weed/filer2/filer_structure.go b/weed/filer2/filer_structure.go index b2c1f5902..1d2da752d 100644 --- a/weed/filer2/filer_structure.go +++ b/weed/filer2/filer_structure.go @@ -5,9 +5,9 @@ import ( "os" "time" "path/filepath" + "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" ) -type FileId string //file id in SeaweedFS type FullPath string func NewFullPath(dir, name string) FullPath { @@ -51,18 +51,12 @@ type Entry struct { Attr // the following is for files - Chunks []FileChunk `json:"chunks,omitempty"` -} - -type FileChunk struct { - Fid FileId `json:"fid,omitempty"` - Offset int64 `json:"offset,omitempty"` - Size uint64 `json:"size,omitempty"` // size in bytes + Chunks []*filer_pb.FileChunk `json:"chunks,omitempty"` } type AbstractFiler interface { CreateEntry(*Entry) (error) - AppendFileChunk(FullPath, FileChunk) (err error) + AppendFileChunk(FullPath, []*filer_pb.FileChunk) (err error) FindEntry(FullPath) (found bool, fileEntry *Entry, err error) DeleteEntry(FullPath) (fileEntry *Entry, err error) @@ -74,7 +68,7 @@ var ErrNotFound = errors.New("filer: no entry is found in filer store") type FilerStore interface { InsertEntry(*Entry) (error) - AppendFileChunk(FullPath, FileChunk) (err error) + AppendFileChunk(FullPath, []*filer_pb.FileChunk) (err error) FindEntry(FullPath) (found bool, entry *Entry, err error) DeleteEntry(FullPath) (fileEntry *Entry, err error) diff --git a/weed/filer2/memdb/memdb_store.go b/weed/filer2/memdb/memdb_store.go index edddcca96..0276e9a7f 100644 --- a/weed/filer2/memdb/memdb_store.go +++ b/weed/filer2/memdb/memdb_store.go @@ -6,6 +6,7 @@ import ( "strings" "fmt" "time" + "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" ) type MemDbStore struct { @@ -32,13 +33,15 @@ func (filer *MemDbStore) InsertEntry(entry *filer2.Entry) (err error) { return nil } -func (filer *MemDbStore) AppendFileChunk(fullpath filer2.FullPath, fileChunk filer2.FileChunk) (err error) { +func (filer *MemDbStore) AppendFileChunk(fullpath filer2.FullPath, fileChunks []*filer_pb.FileChunk) (err error) { found, entry, err := filer.FindEntry(fullpath) if !found { return fmt.Errorf("No such file: %s", fullpath) } - entry.Chunks = append(entry.Chunks, fileChunk) + entry.Chunks = append(entry.Chunks, fileChunks...) entry.Mtime = time.Now() + println("appending to entry", entry.Name(), len(entry.Chunks)) + filer.tree.ReplaceOrInsert(Entry{entry}) return nil } diff --git a/weed/filesys/dir.go b/weed/filesys/dir.go index d408dc1ac..f0191533d 100644 --- a/weed/filesys/dir.go +++ b/weed/filesys/dir.go @@ -9,7 +9,6 @@ import ( "bazil.org/fuse/fs" "bazil.org/fuse" - "github.com/chrislusf/seaweedfs/weed/filer" "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "time" @@ -27,16 +26,77 @@ func (dir *Dir) Attr(context context.Context, attr *fuse.Attr) error { return nil } +func (dir *Dir) Create(ctx context.Context, req *fuse.CreateRequest, + resp *fuse.CreateResponse) (fs.Node, fs.Handle, error) { + + err := dir.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error { + + request := &filer_pb.CreateEntryRequest{ + Directory: dir.Path, + Entry: &filer_pb.Entry{ + Name: req.Name, + IsDirectory: req.Mode&os.ModeDir > 0, + Attributes: &filer_pb.FuseAttributes{ + Mtime: time.Now().Unix(), + FileMode: uint32(req.Mode), + Uid: req.Uid, + Gid: req.Gid, + }, + }, + } + + glog.V(1).Infof("create: %v", request) + if _, err := client.CreateEntry(ctx, request); err != nil { + return fmt.Errorf("create file: %v", err) + } + + return nil + }) + + if err == nil { + node := &File{Name: req.Name, dir: dir, wfs: dir.wfs} + dir.NodeMap[req.Name] = node + return node, node, nil + } + + return nil, nil, err +} + func (dir *Dir) Mkdir(ctx context.Context, req *fuse.MkdirRequest) (fs.Node, error) { dir.NodeMapLock.Lock() defer dir.NodeMapLock.Unlock() - fmt.Printf("mkdir %+v\n", req) + err := dir.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error { - node := &Dir{Path: path.Join(dir.Path, req.Name), wfs: dir.wfs} - dir.NodeMap[req.Name] = node + request := &filer_pb.CreateEntryRequest{ + Directory: dir.Path, + Entry: &filer_pb.Entry{ + Name: req.Name, + IsDirectory: true, + Attributes: &filer_pb.FuseAttributes{ + Mtime: time.Now().Unix(), + FileMode: uint32(req.Mode), + Uid: req.Uid, + Gid: req.Gid, + }, + }, + } - return node, nil + glog.V(1).Infof("mkdir: %v", request) + if _, err := client.CreateEntry(ctx, request); err != nil { + return fmt.Errorf("make dir: %v", err) + } + + return nil + }) + + if err == nil { + node := &Dir{Path: path.Join(dir.Path, req.Name), wfs: dir.wfs} + dir.NodeMap[req.Name] = node + return node, nil + } + + return nil, err } func (dir *Dir) Lookup(ctx context.Context, name string) (node fs.Node, err error) { @@ -75,13 +135,13 @@ func (dir *Dir) Lookup(ctx context.Context, name string) (node fs.Node, err erro if entry.IsDirectory { node = &Dir{Path: path.Join(dir.Path, name), wfs: dir.wfs} } else { - node = &File{FileId: filer.FileId(entry.FileId), Name: name, dir: dir, wfs: dir.wfs} + node = &File{Chunks: entry.Chunks, Name: name, dir: dir, wfs: dir.wfs} } dir.NodeMap[name] = node return node, nil } - return nil, err + return nil, fuse.ENOENT } func (dir *Dir) ReadDirAll(ctx context.Context) (ret []fuse.Dirent, err error) { diff --git a/weed/filesys/file.go b/weed/filesys/file.go index b4a447fa6..160693c54 100644 --- a/weed/filesys/file.go +++ b/weed/filesys/file.go @@ -5,13 +5,14 @@ import ( "fmt" "bazil.org/fuse" - "github.com/chrislusf/seaweedfs/weed/filer" "bazil.org/fuse/fs" "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "path/filepath" "os" "time" + "bytes" + "github.com/chrislusf/seaweedfs/weed/operation" ) var _ = fs.Node(&File{}) @@ -20,10 +21,11 @@ var _ = fs.Node(&File{}) var _ = fs.Handle(&File{}) var _ = fs.HandleReadAller(&File{}) // var _ = fs.HandleReader(&File{}) +var _ = fs.HandleFlusher(&File{}) var _ = fs.HandleWriter(&File{}) type File struct { - FileId filer.FileId + Chunks []*filer_pb.FileChunk Name string dir *Dir wfs *WFS @@ -71,10 +73,15 @@ func (file *File) Attr(context context.Context, attr *fuse.Attr) error { func (file *File) ReadAll(ctx context.Context) (content []byte, err error) { + if len(file.Chunks) == 0 { + return + } + err = file.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error { + // FIXME: need to either use Read() or implement differently request := &filer_pb.GetFileContentRequest{ - FileId: string(file.FileId), + FileId: file.Chunks[0].FileId, } glog.V(1).Infof("read file content: %v", request) @@ -91,7 +98,75 @@ func (file *File) ReadAll(ctx context.Context) (content []byte, err error) { return content, err } +func (file *File) Flush(ctx context.Context, req *fuse.FlushRequest) error { + // write the file chunks to the filer + fmt.Printf("flush file %+v\n", req) + + err := file.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error { + + request := &filer_pb.AppendFileChunksRequest{ + Directory: file.dir.Path, + Entry: &filer_pb.Entry{ + Name: file.Name, + Chunks: file.Chunks, + }, + } + + glog.V(1).Infof("append chunks: %v", request) + if _, err := client.AppendFileChunks(ctx, request); err != nil { + return fmt.Errorf("create file: %v", err) + } + + return nil + }) + + return err +} + func (file *File) Write(ctx context.Context, req *fuse.WriteRequest, resp *fuse.WriteResponse) error { + // write the request to volume servers fmt.Printf("write file %+v\n", req) + + var fileId, host string + + if err := file.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error { + + request := &filer_pb.AssignVolumeRequest{ + Count: 1, + Replication: "000", + Collection: "", + } + + glog.V(1).Infof("assign volume: %v", request) + resp, err := client.AssignVolume(ctx, request) + if err != nil { + return err + } + + fileId, host = resp.FileId, resp.Url + + return nil + }); err != nil { + return fmt.Errorf("filer assign volume: %v", err) + } + + fileUrl := fmt.Sprintf("http://%s/%s", host, fileId) + bufReader := bytes.NewReader(req.Data) + uploadResult, err := operation.Upload(fileUrl, file.Name, bufReader, false, "application/octet-stream", nil, "") + if err != nil { + return fmt.Errorf("upload data: %v", err) + } + if uploadResult.Error != "" { + return fmt.Errorf("upload result: %v", uploadResult.Error) + } + + glog.V(1).Infof("uploaded %s/%s to: %v", file.dir.Path, file.Name, fileUrl) + + file.Chunks = append(file.Chunks, &filer_pb.FileChunk{ + FileId: fileId, + Offset: req.Offset, + Size: uint64(uploadResult.Size), + }) + return nil } diff --git a/weed/operation/assign_file_id.go b/weed/operation/assign_file_id.go index f2365890c..a3466bdd2 100644 --- a/weed/operation/assign_file_id.go +++ b/weed/operation/assign_file_id.go @@ -52,7 +52,7 @@ func Assign(server string, r *VolumeAssignRequest) (*AssignResult, error) { } jsonBlob, err := util.Post("http://"+server+"/dir/assign", values) - glog.V(2).Info("assign result :", string(jsonBlob)) + glog.V(2).Infof("assign result from %s : %s", server, string(jsonBlob)) if err != nil { return nil, err } diff --git a/weed/pb/filer.proto b/weed/pb/filer.proto index 822bf143e..74b7efcd0 100644 --- a/weed/pb/filer.proto +++ b/weed/pb/filer.proto @@ -18,9 +18,18 @@ service SeaweedFiler { rpc GetFileContent (GetFileContentRequest) returns (GetFileContentResponse) { } + rpc CreateEntry (CreateEntryRequest) returns (CreateEntryResponse) { + } + + rpc AppendFileChunks (AppendFileChunksRequest) returns (AppendFileChunksResponse) { + } + rpc DeleteEntry (DeleteEntryRequest) returns (DeleteEntryResponse) { } + rpc AssignVolume (AssignVolumeRequest) returns (AssignVolumeResponse) { + } + } ////////////////////////////////////////////////// @@ -45,10 +54,16 @@ message ListEntriesResponse { message Entry { string name = 1; bool is_directory = 2; - string file_id = 3; + repeated FileChunk chunks = 3; FuseAttributes attributes = 4; } +message FileChunk { + string file_id = 1; + int64 offset = 2; + uint64 size = 3; +} + message FuseAttributes { uint64 file_size = 1; int64 mtime = 2; @@ -75,6 +90,14 @@ message GetFileContentResponse { bytes content = 1; } +message CreateEntryRequest { + string directory = 1; + Entry entry = 2; +} + +message CreateEntryResponse { +} + message DeleteEntryRequest { string directory = 1; string name = 2; @@ -83,3 +106,23 @@ message DeleteEntryRequest { message DeleteEntryResponse { } + +message AssignVolumeRequest { + int32 count = 1; + string collection = 2; + string replication = 3; +} + +message AssignVolumeResponse { + string file_id = 1; + string url = 2; + string public_url = 3; + int32 count = 4; +} + +message AppendFileChunksRequest { + string directory = 1; + Entry entry = 2; +} +message AppendFileChunksResponse { +} diff --git a/weed/pb/filer_pb/filer.pb.go b/weed/pb/filer_pb/filer.pb.go index 98d2c32b6..369fa0826 100644 --- a/weed/pb/filer_pb/filer.pb.go +++ b/weed/pb/filer_pb/filer.pb.go @@ -14,13 +14,20 @@ It has these top-level messages: ListEntriesRequest ListEntriesResponse Entry + FileChunk FuseAttributes GetFileAttributesRequest GetFileAttributesResponse GetFileContentRequest GetFileContentResponse + CreateEntryRequest + CreateEntryResponse DeleteEntryRequest DeleteEntryResponse + AssignVolumeRequest + AssignVolumeResponse + AppendFileChunksRequest + AppendFileChunksResponse */ package filer_pb @@ -119,7 +126,7 @@ func (m *ListEntriesResponse) GetEntries() []*Entry { type Entry struct { Name string `protobuf:"bytes,1,opt,name=name" json:"name,omitempty"` IsDirectory bool `protobuf:"varint,2,opt,name=is_directory,json=isDirectory" json:"is_directory,omitempty"` - FileId string `protobuf:"bytes,3,opt,name=file_id,json=fileId" json:"file_id,omitempty"` + Chunks []*FileChunk `protobuf:"bytes,3,rep,name=chunks" json:"chunks,omitempty"` Attributes *FuseAttributes `protobuf:"bytes,4,opt,name=attributes" json:"attributes,omitempty"` } @@ -142,11 +149,11 @@ func (m *Entry) GetIsDirectory() bool { return false } -func (m *Entry) GetFileId() string { +func (m *Entry) GetChunks() []*FileChunk { if m != nil { - return m.FileId + return m.Chunks } - return "" + return nil } func (m *Entry) GetAttributes() *FuseAttributes { @@ -156,6 +163,38 @@ func (m *Entry) GetAttributes() *FuseAttributes { return nil } +type FileChunk struct { + FileId string `protobuf:"bytes,1,opt,name=file_id,json=fileId" json:"file_id,omitempty"` + Offset int64 `protobuf:"varint,2,opt,name=offset" json:"offset,omitempty"` + Size uint64 `protobuf:"varint,3,opt,name=size" json:"size,omitempty"` +} + +func (m *FileChunk) Reset() { *m = FileChunk{} } +func (m *FileChunk) String() string { return proto.CompactTextString(m) } +func (*FileChunk) ProtoMessage() {} +func (*FileChunk) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{5} } + +func (m *FileChunk) GetFileId() string { + if m != nil { + return m.FileId + } + return "" +} + +func (m *FileChunk) GetOffset() int64 { + if m != nil { + return m.Offset + } + return 0 +} + +func (m *FileChunk) GetSize() uint64 { + if m != nil { + return m.Size + } + return 0 +} + type FuseAttributes struct { FileSize uint64 `protobuf:"varint,1,opt,name=file_size,json=fileSize" json:"file_size,omitempty"` Mtime int64 `protobuf:"varint,2,opt,name=mtime" json:"mtime,omitempty"` @@ -167,7 +206,7 @@ type FuseAttributes struct { func (m *FuseAttributes) Reset() { *m = FuseAttributes{} } func (m *FuseAttributes) String() string { return proto.CompactTextString(m) } func (*FuseAttributes) ProtoMessage() {} -func (*FuseAttributes) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{5} } +func (*FuseAttributes) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{6} } func (m *FuseAttributes) GetFileSize() uint64 { if m != nil { @@ -213,7 +252,7 @@ type GetFileAttributesRequest struct { func (m *GetFileAttributesRequest) Reset() { *m = GetFileAttributesRequest{} } func (m *GetFileAttributesRequest) String() string { return proto.CompactTextString(m) } func (*GetFileAttributesRequest) ProtoMessage() {} -func (*GetFileAttributesRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{6} } +func (*GetFileAttributesRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{7} } func (m *GetFileAttributesRequest) GetName() string { if m != nil { @@ -243,7 +282,7 @@ type GetFileAttributesResponse struct { func (m *GetFileAttributesResponse) Reset() { *m = GetFileAttributesResponse{} } func (m *GetFileAttributesResponse) String() string { return proto.CompactTextString(m) } func (*GetFileAttributesResponse) ProtoMessage() {} -func (*GetFileAttributesResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{7} } +func (*GetFileAttributesResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{8} } func (m *GetFileAttributesResponse) GetAttributes() *FuseAttributes { if m != nil { @@ -259,7 +298,7 @@ type GetFileContentRequest struct { func (m *GetFileContentRequest) Reset() { *m = GetFileContentRequest{} } func (m *GetFileContentRequest) String() string { return proto.CompactTextString(m) } func (*GetFileContentRequest) ProtoMessage() {} -func (*GetFileContentRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{8} } +func (*GetFileContentRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{9} } func (m *GetFileContentRequest) GetFileId() string { if m != nil { @@ -275,7 +314,7 @@ type GetFileContentResponse struct { func (m *GetFileContentResponse) Reset() { *m = GetFileContentResponse{} } func (m *GetFileContentResponse) String() string { return proto.CompactTextString(m) } func (*GetFileContentResponse) ProtoMessage() {} -func (*GetFileContentResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{9} } +func (*GetFileContentResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{10} } func (m *GetFileContentResponse) GetContent() []byte { if m != nil { @@ -284,6 +323,38 @@ func (m *GetFileContentResponse) GetContent() []byte { return nil } +type CreateEntryRequest struct { + Directory string `protobuf:"bytes,1,opt,name=directory" json:"directory,omitempty"` + Entry *Entry `protobuf:"bytes,2,opt,name=entry" json:"entry,omitempty"` +} + +func (m *CreateEntryRequest) Reset() { *m = CreateEntryRequest{} } +func (m *CreateEntryRequest) String() string { return proto.CompactTextString(m) } +func (*CreateEntryRequest) ProtoMessage() {} +func (*CreateEntryRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{11} } + +func (m *CreateEntryRequest) GetDirectory() string { + if m != nil { + return m.Directory + } + return "" +} + +func (m *CreateEntryRequest) GetEntry() *Entry { + if m != nil { + return m.Entry + } + return nil +} + +type CreateEntryResponse struct { +} + +func (m *CreateEntryResponse) Reset() { *m = CreateEntryResponse{} } +func (m *CreateEntryResponse) String() string { return proto.CompactTextString(m) } +func (*CreateEntryResponse) ProtoMessage() {} +func (*CreateEntryResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{12} } + type DeleteEntryRequest struct { Directory string `protobuf:"bytes,1,opt,name=directory" json:"directory,omitempty"` Name string `protobuf:"bytes,2,opt,name=name" json:"name,omitempty"` @@ -293,7 +364,7 @@ type DeleteEntryRequest struct { func (m *DeleteEntryRequest) Reset() { *m = DeleteEntryRequest{} } func (m *DeleteEntryRequest) String() string { return proto.CompactTextString(m) } func (*DeleteEntryRequest) ProtoMessage() {} -func (*DeleteEntryRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{10} } +func (*DeleteEntryRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{13} } func (m *DeleteEntryRequest) GetDirectory() string { if m != nil { @@ -322,7 +393,111 @@ type DeleteEntryResponse struct { func (m *DeleteEntryResponse) Reset() { *m = DeleteEntryResponse{} } func (m *DeleteEntryResponse) String() string { return proto.CompactTextString(m) } func (*DeleteEntryResponse) ProtoMessage() {} -func (*DeleteEntryResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{11} } +func (*DeleteEntryResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{14} } + +type AssignVolumeRequest struct { + Count int32 `protobuf:"varint,1,opt,name=count" json:"count,omitempty"` + Collection string `protobuf:"bytes,2,opt,name=collection" json:"collection,omitempty"` + Replication string `protobuf:"bytes,3,opt,name=replication" json:"replication,omitempty"` +} + +func (m *AssignVolumeRequest) Reset() { *m = AssignVolumeRequest{} } +func (m *AssignVolumeRequest) String() string { return proto.CompactTextString(m) } +func (*AssignVolumeRequest) ProtoMessage() {} +func (*AssignVolumeRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{15} } + +func (m *AssignVolumeRequest) GetCount() int32 { + if m != nil { + return m.Count + } + return 0 +} + +func (m *AssignVolumeRequest) GetCollection() string { + if m != nil { + return m.Collection + } + return "" +} + +func (m *AssignVolumeRequest) GetReplication() string { + if m != nil { + return m.Replication + } + return "" +} + +type AssignVolumeResponse struct { + FileId string `protobuf:"bytes,1,opt,name=file_id,json=fileId" json:"file_id,omitempty"` + Url string `protobuf:"bytes,2,opt,name=url" json:"url,omitempty"` + PublicUrl string `protobuf:"bytes,3,opt,name=public_url,json=publicUrl" json:"public_url,omitempty"` + Count int32 `protobuf:"varint,4,opt,name=count" json:"count,omitempty"` +} + +func (m *AssignVolumeResponse) Reset() { *m = AssignVolumeResponse{} } +func (m *AssignVolumeResponse) String() string { return proto.CompactTextString(m) } +func (*AssignVolumeResponse) ProtoMessage() {} +func (*AssignVolumeResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{16} } + +func (m *AssignVolumeResponse) GetFileId() string { + if m != nil { + return m.FileId + } + return "" +} + +func (m *AssignVolumeResponse) GetUrl() string { + if m != nil { + return m.Url + } + return "" +} + +func (m *AssignVolumeResponse) GetPublicUrl() string { + if m != nil { + return m.PublicUrl + } + return "" +} + +func (m *AssignVolumeResponse) GetCount() int32 { + if m != nil { + return m.Count + } + return 0 +} + +type AppendFileChunksRequest struct { + Directory string `protobuf:"bytes,1,opt,name=directory" json:"directory,omitempty"` + Entry *Entry `protobuf:"bytes,2,opt,name=entry" json:"entry,omitempty"` +} + +func (m *AppendFileChunksRequest) Reset() { *m = AppendFileChunksRequest{} } +func (m *AppendFileChunksRequest) String() string { return proto.CompactTextString(m) } +func (*AppendFileChunksRequest) ProtoMessage() {} +func (*AppendFileChunksRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{17} } + +func (m *AppendFileChunksRequest) GetDirectory() string { + if m != nil { + return m.Directory + } + return "" +} + +func (m *AppendFileChunksRequest) GetEntry() *Entry { + if m != nil { + return m.Entry + } + return nil +} + +type AppendFileChunksResponse struct { +} + +func (m *AppendFileChunksResponse) Reset() { *m = AppendFileChunksResponse{} } +func (m *AppendFileChunksResponse) String() string { return proto.CompactTextString(m) } +func (*AppendFileChunksResponse) ProtoMessage() {} +func (*AppendFileChunksResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{18} } func init() { proto.RegisterType((*LookupDirectoryEntryRequest)(nil), "filer_pb.LookupDirectoryEntryRequest") @@ -330,13 +505,20 @@ func init() { proto.RegisterType((*ListEntriesRequest)(nil), "filer_pb.ListEntriesRequest") proto.RegisterType((*ListEntriesResponse)(nil), "filer_pb.ListEntriesResponse") proto.RegisterType((*Entry)(nil), "filer_pb.Entry") + proto.RegisterType((*FileChunk)(nil), "filer_pb.FileChunk") proto.RegisterType((*FuseAttributes)(nil), "filer_pb.FuseAttributes") proto.RegisterType((*GetFileAttributesRequest)(nil), "filer_pb.GetFileAttributesRequest") proto.RegisterType((*GetFileAttributesResponse)(nil), "filer_pb.GetFileAttributesResponse") proto.RegisterType((*GetFileContentRequest)(nil), "filer_pb.GetFileContentRequest") proto.RegisterType((*GetFileContentResponse)(nil), "filer_pb.GetFileContentResponse") + proto.RegisterType((*CreateEntryRequest)(nil), "filer_pb.CreateEntryRequest") + proto.RegisterType((*CreateEntryResponse)(nil), "filer_pb.CreateEntryResponse") proto.RegisterType((*DeleteEntryRequest)(nil), "filer_pb.DeleteEntryRequest") proto.RegisterType((*DeleteEntryResponse)(nil), "filer_pb.DeleteEntryResponse") + proto.RegisterType((*AssignVolumeRequest)(nil), "filer_pb.AssignVolumeRequest") + proto.RegisterType((*AssignVolumeResponse)(nil), "filer_pb.AssignVolumeResponse") + proto.RegisterType((*AppendFileChunksRequest)(nil), "filer_pb.AppendFileChunksRequest") + proto.RegisterType((*AppendFileChunksResponse)(nil), "filer_pb.AppendFileChunksResponse") } // Reference imports to suppress errors if they are not otherwise used. @@ -354,7 +536,10 @@ type SeaweedFilerClient interface { ListEntries(ctx context.Context, in *ListEntriesRequest, opts ...grpc.CallOption) (*ListEntriesResponse, error) GetFileAttributes(ctx context.Context, in *GetFileAttributesRequest, opts ...grpc.CallOption) (*GetFileAttributesResponse, error) GetFileContent(ctx context.Context, in *GetFileContentRequest, opts ...grpc.CallOption) (*GetFileContentResponse, error) + CreateEntry(ctx context.Context, in *CreateEntryRequest, opts ...grpc.CallOption) (*CreateEntryResponse, error) + AppendFileChunks(ctx context.Context, in *AppendFileChunksRequest, opts ...grpc.CallOption) (*AppendFileChunksResponse, error) DeleteEntry(ctx context.Context, in *DeleteEntryRequest, opts ...grpc.CallOption) (*DeleteEntryResponse, error) + AssignVolume(ctx context.Context, in *AssignVolumeRequest, opts ...grpc.CallOption) (*AssignVolumeResponse, error) } type seaweedFilerClient struct { @@ -401,6 +586,24 @@ func (c *seaweedFilerClient) GetFileContent(ctx context.Context, in *GetFileCont return out, nil } +func (c *seaweedFilerClient) CreateEntry(ctx context.Context, in *CreateEntryRequest, opts ...grpc.CallOption) (*CreateEntryResponse, error) { + out := new(CreateEntryResponse) + err := grpc.Invoke(ctx, "/filer_pb.SeaweedFiler/CreateEntry", in, out, c.cc, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *seaweedFilerClient) AppendFileChunks(ctx context.Context, in *AppendFileChunksRequest, opts ...grpc.CallOption) (*AppendFileChunksResponse, error) { + out := new(AppendFileChunksResponse) + err := grpc.Invoke(ctx, "/filer_pb.SeaweedFiler/AppendFileChunks", in, out, c.cc, opts...) + if err != nil { + return nil, err + } + return out, nil +} + func (c *seaweedFilerClient) DeleteEntry(ctx context.Context, in *DeleteEntryRequest, opts ...grpc.CallOption) (*DeleteEntryResponse, error) { out := new(DeleteEntryResponse) err := grpc.Invoke(ctx, "/filer_pb.SeaweedFiler/DeleteEntry", in, out, c.cc, opts...) @@ -410,6 +613,15 @@ func (c *seaweedFilerClient) DeleteEntry(ctx context.Context, in *DeleteEntryReq return out, nil } +func (c *seaweedFilerClient) AssignVolume(ctx context.Context, in *AssignVolumeRequest, opts ...grpc.CallOption) (*AssignVolumeResponse, error) { + out := new(AssignVolumeResponse) + err := grpc.Invoke(ctx, "/filer_pb.SeaweedFiler/AssignVolume", in, out, c.cc, opts...) + if err != nil { + return nil, err + } + return out, nil +} + // Server API for SeaweedFiler service type SeaweedFilerServer interface { @@ -417,7 +629,10 @@ type SeaweedFilerServer interface { ListEntries(context.Context, *ListEntriesRequest) (*ListEntriesResponse, error) GetFileAttributes(context.Context, *GetFileAttributesRequest) (*GetFileAttributesResponse, error) GetFileContent(context.Context, *GetFileContentRequest) (*GetFileContentResponse, error) + CreateEntry(context.Context, *CreateEntryRequest) (*CreateEntryResponse, error) + AppendFileChunks(context.Context, *AppendFileChunksRequest) (*AppendFileChunksResponse, error) DeleteEntry(context.Context, *DeleteEntryRequest) (*DeleteEntryResponse, error) + AssignVolume(context.Context, *AssignVolumeRequest) (*AssignVolumeResponse, error) } func RegisterSeaweedFilerServer(s *grpc.Server, srv SeaweedFilerServer) { @@ -496,6 +711,42 @@ func _SeaweedFiler_GetFileContent_Handler(srv interface{}, ctx context.Context, return interceptor(ctx, in, info, handler) } +func _SeaweedFiler_CreateEntry_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(CreateEntryRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(SeaweedFilerServer).CreateEntry(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/filer_pb.SeaweedFiler/CreateEntry", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(SeaweedFilerServer).CreateEntry(ctx, req.(*CreateEntryRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _SeaweedFiler_AppendFileChunks_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(AppendFileChunksRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(SeaweedFilerServer).AppendFileChunks(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/filer_pb.SeaweedFiler/AppendFileChunks", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(SeaweedFilerServer).AppendFileChunks(ctx, req.(*AppendFileChunksRequest)) + } + return interceptor(ctx, in, info, handler) +} + func _SeaweedFiler_DeleteEntry_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(DeleteEntryRequest) if err := dec(in); err != nil { @@ -514,6 +765,24 @@ func _SeaweedFiler_DeleteEntry_Handler(srv interface{}, ctx context.Context, dec return interceptor(ctx, in, info, handler) } +func _SeaweedFiler_AssignVolume_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(AssignVolumeRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(SeaweedFilerServer).AssignVolume(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/filer_pb.SeaweedFiler/AssignVolume", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(SeaweedFilerServer).AssignVolume(ctx, req.(*AssignVolumeRequest)) + } + return interceptor(ctx, in, info, handler) +} + var _SeaweedFiler_serviceDesc = grpc.ServiceDesc{ ServiceName: "filer_pb.SeaweedFiler", HandlerType: (*SeaweedFilerServer)(nil), @@ -534,10 +803,22 @@ var _SeaweedFiler_serviceDesc = grpc.ServiceDesc{ MethodName: "GetFileContent", Handler: _SeaweedFiler_GetFileContent_Handler, }, + { + MethodName: "CreateEntry", + Handler: _SeaweedFiler_CreateEntry_Handler, + }, + { + MethodName: "AppendFileChunks", + Handler: _SeaweedFiler_AppendFileChunks_Handler, + }, { MethodName: "DeleteEntry", Handler: _SeaweedFiler_DeleteEntry_Handler, }, + { + MethodName: "AssignVolume", + Handler: _SeaweedFiler_AssignVolume_Handler, + }, }, Streams: []grpc.StreamDesc{}, Metadata: "filer.proto", @@ -546,39 +827,53 @@ var _SeaweedFiler_serviceDesc = grpc.ServiceDesc{ func init() { proto.RegisterFile("filer.proto", fileDescriptor0) } var fileDescriptor0 = []byte{ - // 532 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xa4, 0x54, 0x4d, 0x6f, 0xd3, 0x40, - 0x10, 0xad, 0x71, 0xd2, 0x34, 0x93, 0xb4, 0xc0, 0xb4, 0x05, 0x93, 0xa6, 0x22, 0x2c, 0x2a, 0x82, - 0x4b, 0x84, 0xc2, 0x85, 0x23, 0x88, 0xb4, 0x08, 0x29, 0x08, 0xc9, 0x55, 0xaf, 0x44, 0x49, 0x3d, - 0x8d, 0x56, 0x24, 0x76, 0xf0, 0xae, 0x85, 0xda, 0x33, 0x7f, 0x80, 0xbf, 0xc5, 0xaf, 0x42, 0xbb, - 0xeb, 0x8f, 0x35, 0x76, 0x0a, 0x88, 0x9b, 0xf7, 0xcd, 0xbe, 0x99, 0x37, 0x6f, 0x66, 0x0d, 0x9d, - 0x2b, 0xbe, 0xa4, 0x78, 0xb8, 0x8e, 0x23, 0x19, 0xe1, 0x8e, 0x3e, 0x4c, 0xd7, 0x73, 0xf6, 0x09, - 0x8e, 0x26, 0x51, 0xf4, 0x25, 0x59, 0x8f, 0x79, 0x4c, 0x97, 0x32, 0x8a, 0xaf, 0x4f, 0x43, 0x19, - 0x5f, 0xfb, 0xf4, 0x35, 0x21, 0x21, 0xb1, 0x0f, 0xed, 0x20, 0x0b, 0x78, 0xce, 0xc0, 0x79, 0xde, - 0xf6, 0x0b, 0x00, 0x11, 0x1a, 0xe1, 0x6c, 0x45, 0xde, 0x1d, 0x1d, 0xd0, 0xdf, 0xec, 0x14, 0xfa, - 0xf5, 0x09, 0xc5, 0x3a, 0x0a, 0x05, 0xe1, 0x09, 0x34, 0x49, 0x01, 0x3a, 0x5b, 0x67, 0x74, 0x77, - 0x98, 0x49, 0x19, 0x9a, 0x7b, 0x26, 0xca, 0x46, 0x80, 0x13, 0x2e, 0xa4, 0xc2, 0x38, 0x89, 0xbf, - 0x92, 0xc3, 0xde, 0xc0, 0x7e, 0x89, 0x93, 0x56, 0x7c, 0x01, 0x2d, 0x32, 0x90, 0xe7, 0x0c, 0xdc, - 0xba, 0x9a, 0x59, 0x9c, 0xfd, 0x70, 0xa0, 0xa9, 0xa1, 0xbc, 0x35, 0xa7, 0x68, 0x0d, 0x9f, 0x40, - 0x97, 0x8b, 0x69, 0x21, 0x40, 0xb5, 0xbd, 0xe3, 0x77, 0xb8, 0xc8, 0x5b, 0xc5, 0x87, 0xd0, 0x52, - 0xb9, 0xa7, 0x3c, 0xf0, 0x5c, 0xcd, 0xdc, 0x56, 0xc7, 0x0f, 0x01, 0xbe, 0x06, 0x98, 0x49, 0x19, - 0xf3, 0x79, 0x22, 0x49, 0x78, 0x0d, 0xdd, 0xbb, 0x57, 0xe8, 0x38, 0x4b, 0x04, 0xbd, 0xcd, 0xe3, - 0xbe, 0x75, 0x97, 0x7d, 0x77, 0x60, 0xaf, 0x1c, 0xc6, 0x23, 0x68, 0xeb, 0x2a, 0x82, 0xdf, 0x18, - 0x85, 0x0d, 0x5f, 0x4f, 0xf4, 0x9c, 0xdf, 0x10, 0x1e, 0x40, 0x73, 0x25, 0x79, 0x3a, 0x15, 0xd7, - 0x37, 0x87, 0x9c, 0xb2, 0x8a, 0x02, 0xd2, 0xd2, 0x76, 0x0d, 0xe5, 0x63, 0x14, 0x10, 0xde, 0x03, - 0x37, 0xe1, 0x81, 0x56, 0xb5, 0xeb, 0xab, 0x4f, 0x85, 0x2c, 0x78, 0xe0, 0x35, 0x0d, 0xb2, 0xe0, - 0x01, 0xbb, 0x02, 0xef, 0x3d, 0xc9, 0x33, 0xbe, 0xb4, 0x75, 0xa6, 0x63, 0xa9, 0x33, 0xeb, 0x18, - 0x60, 0x3d, 0x8b, 0x29, 0x94, 0xca, 0xb0, 0x74, 0x43, 0xda, 0x06, 0x19, 0xf3, 0x78, 0xa3, 0x51, - 0xec, 0x02, 0x1e, 0xd5, 0xd4, 0x49, 0x47, 0x59, 0x76, 0xd1, 0xf9, 0x07, 0x17, 0x5f, 0xc2, 0x61, - 0x9a, 0xf6, 0x5d, 0x14, 0x4a, 0x0a, 0x65, 0xa6, 0xdd, 0x12, 0xe2, 0x94, 0x84, 0x8c, 0xe0, 0xc1, - 0xef, 0x8c, 0x54, 0x85, 0x07, 0xad, 0x4b, 0x03, 0x69, 0x4a, 0xd7, 0xcf, 0x8e, 0x8c, 0x03, 0x8e, - 0x69, 0x49, 0x92, 0xfe, 0xef, 0x11, 0x55, 0x36, 0xcd, 0xad, 0x6c, 0x1a, 0x3b, 0x84, 0xfd, 0x52, - 0x29, 0xa3, 0x6d, 0xf4, 0xd3, 0x85, 0xee, 0x39, 0xcd, 0xbe, 0x11, 0x05, 0x4a, 0x7a, 0x8c, 0x0b, - 0x38, 0xa8, 0x7b, 0x8f, 0x78, 0x52, 0xd8, 0x76, 0xcb, 0x0f, 0xa0, 0xf7, 0xec, 0x4f, 0xd7, 0x4c, - 0x5d, 0xb6, 0x85, 0x13, 0xe8, 0x58, 0xaf, 0x0f, 0xfb, 0x16, 0xb1, 0xf2, 0x90, 0x7b, 0xc7, 0x1b, - 0xa2, 0x79, 0xb6, 0xcf, 0x70, 0xbf, 0xb2, 0x06, 0xc8, 0x0a, 0xd6, 0xa6, 0x5d, 0xec, 0x3d, 0xbd, - 0xf5, 0x4e, 0x9e, 0xff, 0x02, 0xf6, 0xca, 0xd3, 0xc5, 0xc7, 0x15, 0x62, 0x79, 0x53, 0x7a, 0x83, - 0xcd, 0x17, 0x6c, 0x13, 0xac, 0xa9, 0xd8, 0x26, 0x54, 0xf7, 0xc2, 0x36, 0xa1, 0x66, 0x94, 0x6c, - 0x6b, 0xbe, 0xad, 0xff, 0xd6, 0xaf, 0x7e, 0x05, 0x00, 0x00, 0xff, 0xff, 0x6a, 0x83, 0x8a, 0x32, - 0xbc, 0x05, 0x00, 0x00, + // 762 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xac, 0x56, 0xdd, 0x4e, 0xdb, 0x48, + 0x14, 0xc6, 0x38, 0x09, 0xe4, 0x24, 0xb0, 0xec, 0x24, 0x80, 0x37, 0xfc, 0x6c, 0x98, 0x15, 0x2b, + 0x56, 0x2b, 0xa1, 0x55, 0xf6, 0x66, 0x2f, 0x17, 0x01, 0xbb, 0xaa, 0x44, 0x45, 0x65, 0x44, 0xa5, + 0xaa, 0x12, 0x51, 0x62, 0x9f, 0xa4, 0x23, 0x1c, 0xdb, 0xb5, 0xc7, 0xad, 0xe0, 0xba, 0xaf, 0xd2, + 0x57, 0xe9, 0x73, 0x55, 0x33, 0x9e, 0xd8, 0x63, 0xec, 0xa4, 0x45, 0xea, 0xdd, 0xcc, 0xf9, 0xfd, + 0xce, 0xcf, 0x37, 0x36, 0xb4, 0x26, 0xcc, 0xc3, 0xe8, 0x34, 0x8c, 0x02, 0x1e, 0x90, 0x75, 0x79, + 0x19, 0x86, 0x63, 0x7a, 0x0d, 0x7b, 0x57, 0x41, 0x70, 0x9f, 0x84, 0x17, 0x2c, 0x42, 0x87, 0x07, + 0xd1, 0xc3, 0xa5, 0xcf, 0xa3, 0x07, 0x1b, 0xdf, 0x27, 0x18, 0x73, 0xb2, 0x0f, 0x4d, 0x77, 0xae, + 0xb0, 0x8c, 0xbe, 0x71, 0xd2, 0xb4, 0x73, 0x01, 0x21, 0x50, 0xf3, 0x47, 0x33, 0xb4, 0x56, 0xa5, + 0x42, 0x9e, 0xe9, 0x25, 0xec, 0x57, 0x07, 0x8c, 0xc3, 0xc0, 0x8f, 0x91, 0x1c, 0x43, 0x1d, 0x85, + 0x40, 0x46, 0x6b, 0x0d, 0x7e, 0x3a, 0x9d, 0x43, 0x39, 0x4d, 0xed, 0x52, 0x2d, 0x1d, 0x00, 0xb9, + 0x62, 0x31, 0x17, 0x32, 0x86, 0xf1, 0x77, 0xc1, 0xa1, 0xff, 0x42, 0xa7, 0xe0, 0xa3, 0x32, 0xfe, + 0x01, 0x6b, 0x98, 0x8a, 0x2c, 0xa3, 0x6f, 0x56, 0xe5, 0x9c, 0xeb, 0xe9, 0x67, 0x03, 0xea, 0x52, + 0x94, 0x95, 0x66, 0xe4, 0xa5, 0x91, 0x23, 0x68, 0xb3, 0x78, 0x98, 0x03, 0x10, 0x65, 0xaf, 0xdb, + 0x2d, 0x16, 0x67, 0xa5, 0x92, 0x3f, 0xa1, 0xe1, 0xbc, 0x4b, 0xfc, 0xfb, 0xd8, 0x32, 0x65, 0xaa, + 0x4e, 0x9e, 0xea, 0x3f, 0xe6, 0xe1, 0xb9, 0xd0, 0xd9, 0xca, 0x84, 0xfc, 0x03, 0x30, 0xe2, 0x3c, + 0x62, 0xe3, 0x84, 0x63, 0x6c, 0xd5, 0x64, 0x3f, 0x2c, 0xcd, 0x21, 0x89, 0xf1, 0x2c, 0xd3, 0xdb, + 0x9a, 0x2d, 0x7d, 0x05, 0xcd, 0x2c, 0x1c, 0xd9, 0x85, 0x35, 0xe1, 0x33, 0x64, 0xae, 0x42, 0xdb, + 0x10, 0xd7, 0x17, 0x2e, 0xd9, 0x81, 0x46, 0x30, 0x99, 0xc4, 0xc8, 0x25, 0x52, 0xd3, 0x56, 0x37, + 0x51, 0x5b, 0xcc, 0x1e, 0xd1, 0x32, 0xfb, 0xc6, 0x49, 0xcd, 0x96, 0x67, 0xfa, 0xc9, 0x80, 0xcd, + 0x62, 0x42, 0xb2, 0x07, 0x4d, 0x19, 0x57, 0xda, 0x1a, 0xd2, 0x56, 0xee, 0xcd, 0x0d, 0x7b, 0x44, + 0xd2, 0x85, 0xfa, 0x8c, 0x33, 0x35, 0x7b, 0xd3, 0x4e, 0x2f, 0x99, 0xcb, 0x2c, 0x70, 0xd3, 0xf0, + 0x1b, 0xa9, 0xcb, 0xcb, 0xc0, 0x45, 0xb2, 0x05, 0x66, 0xc2, 0x5c, 0x59, 0xe7, 0x86, 0x2d, 0x8e, + 0x42, 0x32, 0x65, 0xae, 0x55, 0x4f, 0x25, 0x53, 0xe6, 0xd2, 0x09, 0x58, 0xff, 0x23, 0x17, 0xb5, + 0x69, 0x95, 0xab, 0xe1, 0x57, 0x8d, 0xe4, 0x00, 0x20, 0x1c, 0x45, 0xe8, 0x73, 0x31, 0x16, 0xb5, + 0x87, 0xcd, 0x54, 0x72, 0xc1, 0x22, 0xbd, 0x35, 0xa6, 0xde, 0x1a, 0x7a, 0x0b, 0xbf, 0x54, 0xe4, + 0x51, 0x0b, 0x53, 0x9c, 0x8b, 0xf1, 0x8c, 0xb9, 0xfc, 0x05, 0xdb, 0x2a, 0xec, 0x79, 0xe0, 0x73, + 0xf4, 0xf9, 0x1c, 0xfb, 0xa2, 0x19, 0xd1, 0x01, 0xec, 0x3c, 0xf5, 0x50, 0x28, 0x2c, 0x58, 0x73, + 0x52, 0x91, 0x74, 0x69, 0xdb, 0xf3, 0x2b, 0x7d, 0x03, 0xe4, 0x3c, 0xc2, 0x11, 0xc7, 0x67, 0x50, + 0x35, 0xa3, 0xdd, 0xea, 0x52, 0xda, 0x6d, 0x43, 0xa7, 0x10, 0x3a, 0xc5, 0x42, 0x19, 0x90, 0x0b, + 0xf4, 0xf0, 0x59, 0x19, 0x2b, 0x1e, 0x87, 0x12, 0x83, 0xcc, 0x12, 0x83, 0x04, 0x82, 0x42, 0x2a, + 0x85, 0x60, 0x06, 0x9d, 0xb3, 0x38, 0x66, 0x53, 0xff, 0x75, 0xe0, 0x25, 0x33, 0x9c, 0x43, 0xe8, + 0x42, 0xdd, 0x09, 0x12, 0xd5, 0xa2, 0xba, 0x9d, 0x5e, 0xc8, 0x21, 0x80, 0x13, 0x78, 0x1e, 0x3a, + 0x9c, 0x05, 0xbe, 0x02, 0xa0, 0x49, 0x48, 0x1f, 0x5a, 0x11, 0x86, 0x1e, 0x73, 0x46, 0xd2, 0x20, + 0x5d, 0x0d, 0x5d, 0x44, 0x3f, 0x40, 0xb7, 0x98, 0x4e, 0x0d, 0x65, 0x21, 0xd7, 0xc4, 0x72, 0x47, + 0x9e, 0xca, 0x25, 0x8e, 0x72, 0x35, 0x93, 0xb1, 0xc7, 0x9c, 0xa1, 0x50, 0x98, 0x6a, 0x35, 0xa5, + 0xe4, 0x36, 0xf2, 0x72, 0xe4, 0x35, 0x0d, 0x39, 0xbd, 0x83, 0xdd, 0xb3, 0x30, 0x44, 0xdf, 0xcd, + 0xe8, 0x1d, 0xff, 0xd0, 0xf9, 0xf6, 0xc0, 0x2a, 0xc7, 0x4f, 0x6b, 0x1b, 0x7c, 0xa9, 0x43, 0xfb, + 0x06, 0x47, 0x1f, 0x11, 0xa5, 0x36, 0x22, 0x53, 0xe8, 0x56, 0x3d, 0xe5, 0xe4, 0x38, 0x0f, 0xbe, + 0xe4, 0xdb, 0xd1, 0xfb, 0xfd, 0x5b, 0x66, 0x6a, 0xb4, 0x2b, 0xe4, 0x0a, 0x5a, 0xda, 0xc3, 0x4d, + 0xf6, 0x35, 0xc7, 0xd2, 0x37, 0xa0, 0x77, 0xb0, 0x40, 0x9b, 0x45, 0xbb, 0x83, 0x9f, 0x4b, 0xdc, + 0x26, 0x34, 0xf7, 0x5a, 0xf4, 0xc0, 0xf4, 0x7e, 0x5b, 0x6a, 0x93, 0xc5, 0xbf, 0x85, 0xcd, 0x22, + 0x65, 0xc9, 0xaf, 0x25, 0xc7, 0x22, 0xfd, 0x7b, 0xfd, 0xc5, 0x06, 0x7a, 0x13, 0x34, 0xea, 0xe9, + 0x4d, 0x28, 0x93, 0x5d, 0x6f, 0x42, 0x15, 0x5f, 0x57, 0xc8, 0x5b, 0xd8, 0x7a, 0x3a, 0x68, 0x72, + 0x94, 0x3b, 0x2d, 0x58, 0xb2, 0x1e, 0x5d, 0x66, 0xa2, 0x43, 0xd5, 0x38, 0xaa, 0x43, 0x2d, 0xbf, + 0x12, 0x3a, 0xd4, 0x2a, 0x62, 0xaf, 0x90, 0x6b, 0x68, 0xeb, 0x5c, 0x23, 0x9a, 0x43, 0x05, 0xe5, + 0x7b, 0x87, 0x8b, 0xd4, 0xf3, 0x80, 0xe3, 0x86, 0xfc, 0xc9, 0xf9, 0xfb, 0x6b, 0x00, 0x00, 0x00, + 0xff, 0xff, 0xfe, 0xa5, 0x31, 0x33, 0xf3, 0x08, 0x00, 0x00, } diff --git a/weed/server/filer_grpc_server.go b/weed/server/filer_grpc_server.go index 4d31782da..46a534ffa 100644 --- a/weed/server/filer_grpc_server.go +++ b/weed/server/filer_grpc_server.go @@ -9,6 +9,8 @@ import ( "github.com/chrislusf/seaweedfs/weed/filer2" "path/filepath" "github.com/chrislusf/seaweedfs/weed/glog" + "time" + "os" ) func (fs *FilerServer) LookupDirectoryEntry(ctx context.Context, req *filer_pb.LookupDirectoryEntryRequest) (*filer_pb.LookupDirectoryEntryResponse, error) { @@ -21,16 +23,11 @@ func (fs *FilerServer) LookupDirectoryEntry(ctx context.Context, req *filer_pb.L return nil, fmt.Errorf("%s not found under %s", req.Name, req.Directory) } - var fileId string - if !entry.IsDirectory() && len(entry.Chunks) > 0 { - fileId = string(entry.Chunks[0].Fid) - } - return &filer_pb.LookupDirectoryEntryResponse{ Entry: &filer_pb.Entry{ Name: req.Name, IsDirectory: entry.IsDirectory(), - FileId: fileId, + Chunks: entry.Chunks, }, }, nil } @@ -44,16 +41,12 @@ func (fs *FilerServer) ListEntries(ctx context.Context, req *filer_pb.ListEntrie resp := &filer_pb.ListEntriesResponse{} for _, entry := range entries { - var fileId string - if !entry.IsDirectory() && len(entry.Chunks) > 0 { - fileId = string(entry.Chunks[0].Fid) - } glog.V(0).Infof("%s attr=%v size=%d", entry.Name(), entry.Attr, filer2.Chunks(entry.Chunks).TotalSize()) resp.Entries = append(resp.Entries, &filer_pb.Entry{ Name: entry.Name(), IsDirectory: entry.IsDirectory(), - FileId: fileId, + Chunks: entry.Chunks, Attributes: &filer_pb.FuseAttributes{ FileSize: filer2.Chunks(entry.Chunks).TotalSize(), Mtime: entry.Mtime.Unix(), @@ -106,15 +99,63 @@ func (fs *FilerServer) GetFileContent(ctx context.Context, req *filer_pb.GetFile }, nil } +func (fs *FilerServer) CreateEntry(ctx context.Context, req *filer_pb.CreateEntryRequest) (resp *filer_pb.CreateEntryResponse, err error) { + err = fs.filer.CreateEntry(&filer2.Entry{ + FullPath: filer2.FullPath(filepath.Join(req.Directory, req.Entry.Name)), + Attr: filer2.Attr{ + Mtime: time.Unix(req.Entry.Attributes.Mtime, 0), + Crtime: time.Unix(req.Entry.Attributes.Mtime, 0), + Mode: os.FileMode(req.Entry.Attributes.FileMode), + Uid: req.Entry.Attributes.Uid, + Gid: req.Entry.Attributes.Gid, + }, + }) + + if err == nil { + } + + return &filer_pb.CreateEntryResponse{}, err +} + +func (fs *FilerServer) AppendFileChunks(ctx context.Context, req *filer_pb.AppendFileChunksRequest) (*filer_pb.AppendFileChunksResponse, error) { + err := fs.filer.AppendFileChunk( + filer2.FullPath(filepath.Join(req.Directory, req.Entry.Name)), + req.Entry.Chunks, + ) + + return &filer_pb.AppendFileChunksResponse{}, err +} + func (fs *FilerServer) DeleteEntry(ctx context.Context, req *filer_pb.DeleteEntryRequest) (resp *filer_pb.DeleteEntryResponse, err error) { entry, err := fs.filer.DeleteEntry(filer2.FullPath(filepath.Join(req.Directory, req.Name))) if err == nil { for _, chunk := range entry.Chunks { - fid := string(chunk.Fid) - if err = operation.DeleteFile(fs.getMasterNode(), fid, fs.jwt(fid)); err != nil { - glog.V(0).Infof("deleting file %s: %v", fid, err) + if err = operation.DeleteFile(fs.getMasterNode(), chunk.FileId, fs.jwt(chunk.FileId)); err != nil { + glog.V(0).Infof("deleting file %s: %v", chunk.FileId, err) } } } return &filer_pb.DeleteEntryResponse{}, err } + +func (fs *FilerServer) AssignVolume(ctx context.Context, req *filer_pb.AssignVolumeRequest) (resp *filer_pb.AssignVolumeResponse, err error) { + + assignResult, err := operation.Assign(fs.master, &operation.VolumeAssignRequest{ + Count: uint64(req.Count), + Replication: req.Replication, + Collection: req.Collection, + }) + if err != nil { + return nil, fmt.Errorf("assign volume: %v", err) + } + if assignResult.Error != "" { + return nil, fmt.Errorf("assign volume result: %v", assignResult.Error) + } + + return &filer_pb.AssignVolumeResponse{ + FileId: assignResult.Fid, + Count: int32(assignResult.Count), + Url: assignResult.Url, + PublicUrl: assignResult.PublicUrl, + }, err +} diff --git a/weed/server/filer_server_handlers_admin.go b/weed/server/filer_server_handlers_admin.go index 5c1b23255..9872dcbd1 100644 --- a/weed/server/filer_server_handlers_admin.go +++ b/weed/server/filer_server_handlers_admin.go @@ -6,6 +6,7 @@ import ( "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/filer2" "strconv" + "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" ) func (fs *FilerServer) registerHandler(w http.ResponseWriter, r *http.Request) { @@ -22,9 +23,9 @@ func (fs *FilerServer) registerHandler(w http.ResponseWriter, r *http.Request) { Attr: filer2.Attr{ Mode: 0660, }, - Chunks: []filer2.FileChunk{{ - Fid: filer2.FileId(fileId), - Size: fileSize, + Chunks: []*filer_pb.FileChunk{{ + FileId: fileId, + Size: fileSize, }}, } err = fs.filer.CreateEntry(entry) diff --git a/weed/server/filer_server_handlers_read.go b/weed/server/filer_server_handlers_read.go index 9d13757be..3517299cf 100644 --- a/weed/server/filer_server_handlers_read.go +++ b/weed/server/filer_server_handlers_read.go @@ -98,7 +98,7 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request, } // FIXME pick the right fid - fileId := string(entry.Chunks[0].Fid) + fileId := entry.Chunks[0].FileId urlLocation, err := operation.LookupFileId(fs.getMasterNode(), fileId) if err != nil { diff --git a/weed/server/filer_server_handlers_write.go b/weed/server/filer_server_handlers_write.go index 1a4b62235..4d988e6a2 100644 --- a/weed/server/filer_server_handlers_write.go +++ b/weed/server/filer_server_handlers_write.go @@ -23,6 +23,7 @@ import ( "github.com/chrislusf/seaweedfs/weed/storage" "github.com/chrislusf/seaweedfs/weed/util" "github.com/chrislusf/seaweedfs/weed/filer2" + "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" ) type FilerPostResult struct { @@ -80,7 +81,7 @@ func (fs *FilerServer) queryFileInfoByPath(w http.ResponseWriter, r *http.Reques glog.V(0).Infoln("failing to find path in filer store", path, err.Error()) writeJsonError(w, r, http.StatusInternalServerError, err) } else if found { - fileId = string(entry.Chunks[0].Fid) + fileId = entry.Chunks[0].FileId urlLocation, err = operation.LookupFileId(fs.getMasterNode(), fileId) if err != nil { glog.V(1).Infoln("operation LookupFileId %s failed, err is %s", fileId, err.Error()) @@ -318,7 +319,7 @@ func (fs *FilerServer) PostHandler(w http.ResponseWriter, r *http.Request) { // also delete the old fid unless PUT operation if r.Method != "PUT" { if found, entry, err := fs.filer.FindEntry(filer2.FullPath(path)); err == nil && found { - oldFid := string(entry.Chunks[0].Fid) + oldFid := entry.Chunks[0].FileId operation.DeleteFile(fs.getMasterNode(), oldFid, fs.jwt(oldFid)) } else if err != nil && err != filer.ErrNotFound { glog.V(0).Infof("error %v occur when finding %s in filer store", err, path) @@ -331,9 +332,9 @@ func (fs *FilerServer) PostHandler(w http.ResponseWriter, r *http.Request) { Attr: filer2.Attr{ Mode: 0660, }, - Chunks: []filer2.FileChunk{{ - Fid: filer2.FileId(fileId), - Size: uint64(r.ContentLength), + Chunks: []*filer_pb.FileChunk{{ + FileId: fileId, + Size: uint64(r.ContentLength), }}, } if db_err := fs.filer.CreateEntry(entry); db_err != nil { @@ -415,7 +416,7 @@ func (fs *FilerServer) doAutoChunk(w http.ResponseWriter, r *http.Request, conte fileName = path.Base(fileName) } - var fileChunks []filer2.FileChunk + var fileChunks []*filer_pb.FileChunk totalBytesRead := int64(0) tmpBufferSize := int32(1024 * 1024) @@ -455,8 +456,8 @@ func (fs *FilerServer) doAutoChunk(w http.ResponseWriter, r *http.Request, conte // Save to chunk manifest structure fileChunks = append(fileChunks, - filer2.FileChunk{ - Fid: filer2.FileId(fileId), + &filer_pb.FileChunk{ + FileId: fileId, Offset: chunkOffset, Size: uint64(chunkBufOffset), }, @@ -483,7 +484,7 @@ func (fs *FilerServer) doAutoChunk(w http.ResponseWriter, r *http.Request, conte if r.Method != "PUT" { if found, entry, err := fs.filer.FindEntry(filer2.FullPath(path)); found && err == nil { for _, chunk := range entry.Chunks { - oldFid := string(chunk.Fid) + oldFid := chunk.FileId operation.DeleteFile(fs.getMasterNode(), oldFid, fs.jwt(oldFid)) } } else if err != nil { @@ -535,7 +536,7 @@ func (fs *FilerServer) DeleteHandler(w http.ResponseWriter, r *http.Request) { if entry != nil && !entry.IsDirectory() { for _, chunk := range entry.Chunks { - oldFid := string(chunk.Fid) + oldFid := chunk.FileId operation.DeleteFile(fs.getMasterNode(), oldFid, fs.jwt(oldFid)) } } From 6bf31467c7918b16440be80495a024b877587363 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Wed, 16 May 2018 00:54:27 -0700 Subject: [PATCH 07/83] adding empty fsync less error now. But still: vi on write: E514: write error (file system full?) cp: Input/output error --- weed/filesys/file.go | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/weed/filesys/file.go b/weed/filesys/file.go index 160693c54..d3fb8283a 100644 --- a/weed/filesys/file.go +++ b/weed/filesys/file.go @@ -17,7 +17,7 @@ import ( var _ = fs.Node(&File{}) // var _ = fs.NodeOpener(&File{}) -// var _ = fs.NodeFsyncer(&File{}) +var _ = fs.NodeFsyncer(&File{}) var _ = fs.Handle(&File{}) var _ = fs.HandleReadAller(&File{}) // var _ = fs.HandleReader(&File{}) @@ -98,10 +98,19 @@ func (file *File) ReadAll(ctx context.Context) (content []byte, err error) { return content, err } -func (file *File) Flush(ctx context.Context, req *fuse.FlushRequest) error { +func (file *File) Fsync(ctx context.Context, req *fuse.FsyncRequest) error { + // fsync works at OS level // write the file chunks to the filer fmt.Printf("flush file %+v\n", req) + return nil +} + +func (file *File) Flush(ctx context.Context, req *fuse.FlushRequest) error { + // fflush works at file level + // send the data to the OS + fmt.Printf("flush file %+v\n", req) + err := file.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error { request := &filer_pb.AppendFileChunksRequest{ From e31c514b00b97c2db20232812899d542e37ea931 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Wed, 16 May 2018 00:54:44 -0700 Subject: [PATCH 08/83] adding modified time to file chunk --- weed/filesys/file.go | 1 + weed/pb/filer.proto | 1 + weed/pb/filer_pb/filer.pb.go | 106 +++++++++++---------- weed/server/filer_server_handlers_admin.go | 2 + weed/server/filer_server_handlers_write.go | 3 + 5 files changed, 64 insertions(+), 49 deletions(-) diff --git a/weed/filesys/file.go b/weed/filesys/file.go index d3fb8283a..18fd05e1d 100644 --- a/weed/filesys/file.go +++ b/weed/filesys/file.go @@ -175,6 +175,7 @@ func (file *File) Write(ctx context.Context, req *fuse.WriteRequest, resp *fuse. FileId: fileId, Offset: req.Offset, Size: uint64(uploadResult.Size), + Mtime: time.Now().UnixNano(), }) return nil diff --git a/weed/pb/filer.proto b/weed/pb/filer.proto index 74b7efcd0..461a76a58 100644 --- a/weed/pb/filer.proto +++ b/weed/pb/filer.proto @@ -62,6 +62,7 @@ message FileChunk { string file_id = 1; int64 offset = 2; uint64 size = 3; + int64 mtime = 4; } message FuseAttributes { diff --git a/weed/pb/filer_pb/filer.pb.go b/weed/pb/filer_pb/filer.pb.go index 369fa0826..fc3bb65c6 100644 --- a/weed/pb/filer_pb/filer.pb.go +++ b/weed/pb/filer_pb/filer.pb.go @@ -167,6 +167,7 @@ type FileChunk struct { FileId string `protobuf:"bytes,1,opt,name=file_id,json=fileId" json:"file_id,omitempty"` Offset int64 `protobuf:"varint,2,opt,name=offset" json:"offset,omitempty"` Size uint64 `protobuf:"varint,3,opt,name=size" json:"size,omitempty"` + Mtime int64 `protobuf:"varint,4,opt,name=mtime" json:"mtime,omitempty"` } func (m *FileChunk) Reset() { *m = FileChunk{} } @@ -195,6 +196,13 @@ func (m *FileChunk) GetSize() uint64 { return 0 } +func (m *FileChunk) GetMtime() int64 { + if m != nil { + return m.Mtime + } + return 0 +} + type FuseAttributes struct { FileSize uint64 `protobuf:"varint,1,opt,name=file_size,json=fileSize" json:"file_size,omitempty"` Mtime int64 `protobuf:"varint,2,opt,name=mtime" json:"mtime,omitempty"` @@ -827,53 +835,53 @@ var _SeaweedFiler_serviceDesc = grpc.ServiceDesc{ func init() { proto.RegisterFile("filer.proto", fileDescriptor0) } var fileDescriptor0 = []byte{ - // 762 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xac, 0x56, 0xdd, 0x4e, 0xdb, 0x48, - 0x14, 0xc6, 0x38, 0x09, 0xe4, 0x24, 0xb0, 0xec, 0x24, 0x80, 0x37, 0xfc, 0x6c, 0x98, 0x15, 0x2b, - 0x56, 0x2b, 0xa1, 0x55, 0xf6, 0x66, 0x2f, 0x17, 0x01, 0xbb, 0xaa, 0x44, 0x45, 0x65, 0x44, 0xa5, - 0xaa, 0x12, 0x51, 0x62, 0x9f, 0xa4, 0x23, 0x1c, 0xdb, 0xb5, 0xc7, 0xad, 0xe0, 0xba, 0xaf, 0xd2, - 0x57, 0xe9, 0x73, 0x55, 0x33, 0x9e, 0xd8, 0x63, 0xec, 0xa4, 0x45, 0xea, 0xdd, 0xcc, 0xf9, 0xfd, - 0xce, 0xcf, 0x37, 0x36, 0xb4, 0x26, 0xcc, 0xc3, 0xe8, 0x34, 0x8c, 0x02, 0x1e, 0x90, 0x75, 0x79, - 0x19, 0x86, 0x63, 0x7a, 0x0d, 0x7b, 0x57, 0x41, 0x70, 0x9f, 0x84, 0x17, 0x2c, 0x42, 0x87, 0x07, - 0xd1, 0xc3, 0xa5, 0xcf, 0xa3, 0x07, 0x1b, 0xdf, 0x27, 0x18, 0x73, 0xb2, 0x0f, 0x4d, 0x77, 0xae, - 0xb0, 0x8c, 0xbe, 0x71, 0xd2, 0xb4, 0x73, 0x01, 0x21, 0x50, 0xf3, 0x47, 0x33, 0xb4, 0x56, 0xa5, - 0x42, 0x9e, 0xe9, 0x25, 0xec, 0x57, 0x07, 0x8c, 0xc3, 0xc0, 0x8f, 0x91, 0x1c, 0x43, 0x1d, 0x85, - 0x40, 0x46, 0x6b, 0x0d, 0x7e, 0x3a, 0x9d, 0x43, 0x39, 0x4d, 0xed, 0x52, 0x2d, 0x1d, 0x00, 0xb9, - 0x62, 0x31, 0x17, 0x32, 0x86, 0xf1, 0x77, 0xc1, 0xa1, 0xff, 0x42, 0xa7, 0xe0, 0xa3, 0x32, 0xfe, - 0x01, 0x6b, 0x98, 0x8a, 0x2c, 0xa3, 0x6f, 0x56, 0xe5, 0x9c, 0xeb, 0xe9, 0x67, 0x03, 0xea, 0x52, - 0x94, 0x95, 0x66, 0xe4, 0xa5, 0x91, 0x23, 0x68, 0xb3, 0x78, 0x98, 0x03, 0x10, 0x65, 0xaf, 0xdb, - 0x2d, 0x16, 0x67, 0xa5, 0x92, 0x3f, 0xa1, 0xe1, 0xbc, 0x4b, 0xfc, 0xfb, 0xd8, 0x32, 0x65, 0xaa, - 0x4e, 0x9e, 0xea, 0x3f, 0xe6, 0xe1, 0xb9, 0xd0, 0xd9, 0xca, 0x84, 0xfc, 0x03, 0x30, 0xe2, 0x3c, - 0x62, 0xe3, 0x84, 0x63, 0x6c, 0xd5, 0x64, 0x3f, 0x2c, 0xcd, 0x21, 0x89, 0xf1, 0x2c, 0xd3, 0xdb, - 0x9a, 0x2d, 0x7d, 0x05, 0xcd, 0x2c, 0x1c, 0xd9, 0x85, 0x35, 0xe1, 0x33, 0x64, 0xae, 0x42, 0xdb, - 0x10, 0xd7, 0x17, 0x2e, 0xd9, 0x81, 0x46, 0x30, 0x99, 0xc4, 0xc8, 0x25, 0x52, 0xd3, 0x56, 0x37, - 0x51, 0x5b, 0xcc, 0x1e, 0xd1, 0x32, 0xfb, 0xc6, 0x49, 0xcd, 0x96, 0x67, 0xfa, 0xc9, 0x80, 0xcd, - 0x62, 0x42, 0xb2, 0x07, 0x4d, 0x19, 0x57, 0xda, 0x1a, 0xd2, 0x56, 0xee, 0xcd, 0x0d, 0x7b, 0x44, - 0xd2, 0x85, 0xfa, 0x8c, 0x33, 0x35, 0x7b, 0xd3, 0x4e, 0x2f, 0x99, 0xcb, 0x2c, 0x70, 0xd3, 0xf0, - 0x1b, 0xa9, 0xcb, 0xcb, 0xc0, 0x45, 0xb2, 0x05, 0x66, 0xc2, 0x5c, 0x59, 0xe7, 0x86, 0x2d, 0x8e, - 0x42, 0x32, 0x65, 0xae, 0x55, 0x4f, 0x25, 0x53, 0xe6, 0xd2, 0x09, 0x58, 0xff, 0x23, 0x17, 0xb5, - 0x69, 0x95, 0xab, 0xe1, 0x57, 0x8d, 0xe4, 0x00, 0x20, 0x1c, 0x45, 0xe8, 0x73, 0x31, 0x16, 0xb5, - 0x87, 0xcd, 0x54, 0x72, 0xc1, 0x22, 0xbd, 0x35, 0xa6, 0xde, 0x1a, 0x7a, 0x0b, 0xbf, 0x54, 0xe4, - 0x51, 0x0b, 0x53, 0x9c, 0x8b, 0xf1, 0x8c, 0xb9, 0xfc, 0x05, 0xdb, 0x2a, 0xec, 0x79, 0xe0, 0x73, - 0xf4, 0xf9, 0x1c, 0xfb, 0xa2, 0x19, 0xd1, 0x01, 0xec, 0x3c, 0xf5, 0x50, 0x28, 0x2c, 0x58, 0x73, - 0x52, 0x91, 0x74, 0x69, 0xdb, 0xf3, 0x2b, 0x7d, 0x03, 0xe4, 0x3c, 0xc2, 0x11, 0xc7, 0x67, 0x50, - 0x35, 0xa3, 0xdd, 0xea, 0x52, 0xda, 0x6d, 0x43, 0xa7, 0x10, 0x3a, 0xc5, 0x42, 0x19, 0x90, 0x0b, - 0xf4, 0xf0, 0x59, 0x19, 0x2b, 0x1e, 0x87, 0x12, 0x83, 0xcc, 0x12, 0x83, 0x04, 0x82, 0x42, 0x2a, - 0x85, 0x60, 0x06, 0x9d, 0xb3, 0x38, 0x66, 0x53, 0xff, 0x75, 0xe0, 0x25, 0x33, 0x9c, 0x43, 0xe8, - 0x42, 0xdd, 0x09, 0x12, 0xd5, 0xa2, 0xba, 0x9d, 0x5e, 0xc8, 0x21, 0x80, 0x13, 0x78, 0x1e, 0x3a, - 0x9c, 0x05, 0xbe, 0x02, 0xa0, 0x49, 0x48, 0x1f, 0x5a, 0x11, 0x86, 0x1e, 0x73, 0x46, 0xd2, 0x20, - 0x5d, 0x0d, 0x5d, 0x44, 0x3f, 0x40, 0xb7, 0x98, 0x4e, 0x0d, 0x65, 0x21, 0xd7, 0xc4, 0x72, 0x47, - 0x9e, 0xca, 0x25, 0x8e, 0x72, 0x35, 0x93, 0xb1, 0xc7, 0x9c, 0xa1, 0x50, 0x98, 0x6a, 0x35, 0xa5, - 0xe4, 0x36, 0xf2, 0x72, 0xe4, 0x35, 0x0d, 0x39, 0xbd, 0x83, 0xdd, 0xb3, 0x30, 0x44, 0xdf, 0xcd, - 0xe8, 0x1d, 0xff, 0xd0, 0xf9, 0xf6, 0xc0, 0x2a, 0xc7, 0x4f, 0x6b, 0x1b, 0x7c, 0xa9, 0x43, 0xfb, - 0x06, 0x47, 0x1f, 0x11, 0xa5, 0x36, 0x22, 0x53, 0xe8, 0x56, 0x3d, 0xe5, 0xe4, 0x38, 0x0f, 0xbe, - 0xe4, 0xdb, 0xd1, 0xfb, 0xfd, 0x5b, 0x66, 0x6a, 0xb4, 0x2b, 0xe4, 0x0a, 0x5a, 0xda, 0xc3, 0x4d, - 0xf6, 0x35, 0xc7, 0xd2, 0x37, 0xa0, 0x77, 0xb0, 0x40, 0x9b, 0x45, 0xbb, 0x83, 0x9f, 0x4b, 0xdc, - 0x26, 0x34, 0xf7, 0x5a, 0xf4, 0xc0, 0xf4, 0x7e, 0x5b, 0x6a, 0x93, 0xc5, 0xbf, 0x85, 0xcd, 0x22, - 0x65, 0xc9, 0xaf, 0x25, 0xc7, 0x22, 0xfd, 0x7b, 0xfd, 0xc5, 0x06, 0x7a, 0x13, 0x34, 0xea, 0xe9, - 0x4d, 0x28, 0x93, 0x5d, 0x6f, 0x42, 0x15, 0x5f, 0x57, 0xc8, 0x5b, 0xd8, 0x7a, 0x3a, 0x68, 0x72, - 0x94, 0x3b, 0x2d, 0x58, 0xb2, 0x1e, 0x5d, 0x66, 0xa2, 0x43, 0xd5, 0x38, 0xaa, 0x43, 0x2d, 0xbf, - 0x12, 0x3a, 0xd4, 0x2a, 0x62, 0xaf, 0x90, 0x6b, 0x68, 0xeb, 0x5c, 0x23, 0x9a, 0x43, 0x05, 0xe5, - 0x7b, 0x87, 0x8b, 0xd4, 0xf3, 0x80, 0xe3, 0x86, 0xfc, 0xc9, 0xf9, 0xfb, 0x6b, 0x00, 0x00, 0x00, - 0xff, 0xff, 0xfe, 0xa5, 0x31, 0x33, 0xf3, 0x08, 0x00, 0x00, + // 767 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xac, 0x56, 0x6d, 0x4f, 0xdb, 0x48, + 0x10, 0xc6, 0x38, 0x09, 0x64, 0x12, 0x38, 0x6e, 0x13, 0xc0, 0x17, 0x5e, 0x2e, 0xec, 0x89, 0x13, + 0xa7, 0x93, 0xd0, 0x29, 0xf7, 0xa5, 0x1f, 0x8b, 0x80, 0x56, 0x95, 0xa8, 0x90, 0x8c, 0xa8, 0x54, + 0x55, 0x22, 0x4a, 0xec, 0x49, 0xba, 0xc2, 0xb1, 0x5d, 0x7b, 0xdd, 0x8a, 0x7e, 0xee, 0x5f, 0xe9, + 0x5f, 0xe9, 0xef, 0xaa, 0x76, 0xbd, 0xb1, 0xd7, 0xd8, 0x49, 0x8b, 0xd4, 0x6f, 0xde, 0x79, 0x79, + 0xe6, 0xd9, 0x99, 0x79, 0x36, 0x81, 0xd6, 0x84, 0x79, 0x18, 0x9d, 0x86, 0x51, 0xc0, 0x03, 0xb2, + 0x2e, 0x0f, 0xc3, 0x70, 0x4c, 0xaf, 0x61, 0xef, 0x2a, 0x08, 0xee, 0x93, 0xf0, 0x82, 0x45, 0xe8, + 0xf0, 0x20, 0x7a, 0xb8, 0xf4, 0x79, 0xf4, 0x60, 0xe3, 0x87, 0x04, 0x63, 0x4e, 0xf6, 0xa1, 0xe9, + 0xce, 0x1d, 0x96, 0xd1, 0x37, 0x4e, 0x9a, 0x76, 0x6e, 0x20, 0x04, 0x6a, 0xfe, 0x68, 0x86, 0xd6, + 0xaa, 0x74, 0xc8, 0x6f, 0x7a, 0x09, 0xfb, 0xd5, 0x80, 0x71, 0x18, 0xf8, 0x31, 0x92, 0x63, 0xa8, + 0xa3, 0x30, 0x48, 0xb4, 0xd6, 0xe0, 0xb7, 0xd3, 0x39, 0x95, 0xd3, 0x34, 0x2e, 0xf5, 0xd2, 0x01, + 0x90, 0x2b, 0x16, 0x73, 0x61, 0x63, 0x18, 0xff, 0x14, 0x1d, 0xfa, 0x1c, 0x3a, 0x85, 0x1c, 0x55, + 0xf1, 0x1f, 0x58, 0xc3, 0xd4, 0x64, 0x19, 0x7d, 0xb3, 0xaa, 0xe6, 0xdc, 0x4f, 0xbf, 0x1a, 0x50, + 0x97, 0xa6, 0xec, 0x6a, 0x46, 0x7e, 0x35, 0x72, 0x04, 0x6d, 0x16, 0x0f, 0x73, 0x02, 0xe2, 0xda, + 0xeb, 0x76, 0x8b, 0xc5, 0xd9, 0x55, 0xc9, 0xbf, 0xd0, 0x70, 0xde, 0x27, 0xfe, 0x7d, 0x6c, 0x99, + 0xb2, 0x54, 0x27, 0x2f, 0xf5, 0x82, 0x79, 0x78, 0x2e, 0x7c, 0xb6, 0x0a, 0x21, 0xcf, 0x00, 0x46, + 0x9c, 0x47, 0x6c, 0x9c, 0x70, 0x8c, 0xad, 0x9a, 0xec, 0x87, 0xa5, 0x25, 0x24, 0x31, 0x9e, 0x65, + 0x7e, 0x5b, 0x8b, 0xa5, 0x13, 0x68, 0x66, 0x70, 0x64, 0x17, 0xd6, 0x44, 0xce, 0x90, 0xb9, 0x8a, + 0x6d, 0x43, 0x1c, 0x5f, 0xb9, 0x64, 0x07, 0x1a, 0xc1, 0x64, 0x12, 0x23, 0x97, 0x4c, 0x4d, 0x5b, + 0x9d, 0xc4, 0xdd, 0x62, 0xf6, 0x19, 0x2d, 0xb3, 0x6f, 0x9c, 0xd4, 0x6c, 0xf9, 0x4d, 0xba, 0x50, + 0x9f, 0x71, 0x36, 0x43, 0x49, 0xc3, 0xb4, 0xd3, 0x03, 0xfd, 0x62, 0xc0, 0x66, 0x91, 0x06, 0xd9, + 0x83, 0xa6, 0xac, 0x26, 0x11, 0x0c, 0x89, 0x20, 0xb7, 0xe9, 0xa6, 0x80, 0xb2, 0xaa, 0xa1, 0x64, + 0x29, 0xb3, 0xc0, 0x4d, 0x8b, 0x6e, 0xa4, 0x29, 0xaf, 0x03, 0x17, 0xc9, 0x16, 0x98, 0x09, 0x73, + 0x65, 0xd9, 0x0d, 0x5b, 0x7c, 0x0a, 0xcb, 0x94, 0xb9, 0x56, 0x3d, 0xb5, 0x4c, 0x99, 0x4b, 0x27, + 0x60, 0xbd, 0x44, 0x2e, 0x6e, 0xac, 0xf5, 0x43, 0xad, 0x44, 0xd5, 0xa0, 0x0e, 0x00, 0xc2, 0x51, + 0x84, 0x3e, 0x17, 0xc3, 0x52, 0xdb, 0xd9, 0x4c, 0x2d, 0x17, 0x2c, 0xd2, 0x1b, 0x66, 0xea, 0x0d, + 0xa3, 0xb7, 0xf0, 0x47, 0x45, 0x1d, 0xb5, 0x46, 0xc5, 0x69, 0x19, 0x4f, 0x98, 0xd6, 0x7f, 0xb0, + 0xad, 0x60, 0xcf, 0x03, 0x9f, 0xa3, 0xcf, 0xe7, 0xdc, 0x17, 0x4d, 0x8e, 0x0e, 0x60, 0xe7, 0x71, + 0x86, 0x62, 0x61, 0xc1, 0x9a, 0x93, 0x9a, 0x64, 0x4a, 0xdb, 0x9e, 0x1f, 0xe9, 0x5b, 0x20, 0xe7, + 0x11, 0x8e, 0x38, 0x3e, 0x41, 0xc0, 0x99, 0x18, 0x57, 0x97, 0x8a, 0x71, 0x1b, 0x3a, 0x05, 0xe8, + 0x94, 0x0b, 0x65, 0x40, 0x2e, 0xd0, 0xc3, 0x27, 0x55, 0xac, 0x78, 0x32, 0x4a, 0xba, 0x32, 0x4b, + 0xba, 0x12, 0x0c, 0x0a, 0xa5, 0x14, 0x83, 0x19, 0x74, 0xce, 0xe2, 0x98, 0x4d, 0xfd, 0x37, 0x81, + 0x97, 0xcc, 0x70, 0x4e, 0xa1, 0x0b, 0x75, 0x27, 0x48, 0x54, 0x8b, 0xea, 0x76, 0x7a, 0x20, 0x87, + 0x00, 0x4e, 0xe0, 0x79, 0xe8, 0x70, 0x16, 0xf8, 0x8a, 0x80, 0x66, 0x21, 0x7d, 0x68, 0x45, 0x18, + 0x7a, 0xcc, 0x19, 0xc9, 0x80, 0x74, 0x35, 0x74, 0x13, 0xfd, 0x08, 0xdd, 0x62, 0x39, 0x35, 0x94, + 0x85, 0x0a, 0x14, 0xcb, 0x1d, 0x79, 0xaa, 0x96, 0xf8, 0x94, 0xab, 0x99, 0x8c, 0x3d, 0xe6, 0x0c, + 0x85, 0xc3, 0x54, 0xab, 0x29, 0x2d, 0xb7, 0x91, 0x97, 0x33, 0xaf, 0x69, 0xcc, 0xe9, 0x1d, 0xec, + 0x9e, 0x85, 0x21, 0xfa, 0x6e, 0x26, 0xfa, 0xf8, 0x97, 0xce, 0xb7, 0x07, 0x56, 0x19, 0x3f, 0xbd, + 0xdb, 0xe0, 0x5b, 0x1d, 0xda, 0x37, 0x38, 0xfa, 0x84, 0x28, 0xbd, 0x11, 0x99, 0x42, 0xb7, 0xea, + 0x81, 0x27, 0xc7, 0x39, 0xf8, 0x92, 0x5f, 0x94, 0xde, 0xdf, 0x3f, 0x0a, 0x53, 0xa3, 0x5d, 0x21, + 0x57, 0xd0, 0xd2, 0x9e, 0x73, 0xb2, 0xaf, 0x25, 0x96, 0x7e, 0x19, 0x7a, 0x07, 0x0b, 0xbc, 0x19, + 0xda, 0x1d, 0xfc, 0x5e, 0xd2, 0x36, 0xa1, 0x79, 0xd6, 0xa2, 0x07, 0xa6, 0xf7, 0xd7, 0xd2, 0x98, + 0x0c, 0xff, 0x16, 0x36, 0x8b, 0x92, 0x25, 0x7f, 0x96, 0x12, 0x8b, 0xf2, 0xef, 0xf5, 0x17, 0x07, + 0xe8, 0x4d, 0xd0, 0xa4, 0xa7, 0x37, 0xa1, 0x2c, 0x76, 0xbd, 0x09, 0x55, 0x7a, 0x5d, 0x21, 0xef, + 0x60, 0xeb, 0xf1, 0xa0, 0xc9, 0x51, 0x9e, 0xb4, 0x60, 0xc9, 0x7a, 0x74, 0x59, 0x88, 0x4e, 0x55, + 0xd3, 0xa8, 0x4e, 0xb5, 0xfc, 0x4a, 0xe8, 0x54, 0xab, 0x84, 0xbd, 0x42, 0xae, 0xa1, 0xad, 0x6b, + 0x8d, 0x68, 0x09, 0x15, 0x92, 0xef, 0x1d, 0x2e, 0x72, 0xcf, 0x01, 0xc7, 0x0d, 0xf9, 0xd7, 0xe7, + 0xff, 0xef, 0x01, 0x00, 0x00, 0xff, 0xff, 0x01, 0x52, 0xae, 0x82, 0x09, 0x09, 0x00, 0x00, } diff --git a/weed/server/filer_server_handlers_admin.go b/weed/server/filer_server_handlers_admin.go index 9872dcbd1..2c5c1bc21 100644 --- a/weed/server/filer_server_handlers_admin.go +++ b/weed/server/filer_server_handlers_admin.go @@ -7,6 +7,7 @@ import ( "github.com/chrislusf/seaweedfs/weed/filer2" "strconv" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" + "time" ) func (fs *FilerServer) registerHandler(w http.ResponseWriter, r *http.Request) { @@ -26,6 +27,7 @@ func (fs *FilerServer) registerHandler(w http.ResponseWriter, r *http.Request) { Chunks: []*filer_pb.FileChunk{{ FileId: fileId, Size: fileSize, + Mtime: time.Now().UnixNano(), }}, } err = fs.filer.CreateEntry(entry) diff --git a/weed/server/filer_server_handlers_write.go b/weed/server/filer_server_handlers_write.go index 4d988e6a2..c6e735cf5 100644 --- a/weed/server/filer_server_handlers_write.go +++ b/weed/server/filer_server_handlers_write.go @@ -24,6 +24,7 @@ import ( "github.com/chrislusf/seaweedfs/weed/util" "github.com/chrislusf/seaweedfs/weed/filer2" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" + "time" ) type FilerPostResult struct { @@ -335,6 +336,7 @@ func (fs *FilerServer) PostHandler(w http.ResponseWriter, r *http.Request) { Chunks: []*filer_pb.FileChunk{{ FileId: fileId, Size: uint64(r.ContentLength), + Mtime: time.Now().UnixNano(), }}, } if db_err := fs.filer.CreateEntry(entry); db_err != nil { @@ -460,6 +462,7 @@ func (fs *FilerServer) doAutoChunk(w http.ResponseWriter, r *http.Request, conte FileId: fileId, Offset: chunkOffset, Size: uint64(chunkBufOffset), + Mtime: time.Now().UnixNano(), }, ) From ab4ddb1e0ee425d0d779eb529c42dc5361320822 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sat, 19 May 2018 12:07:15 -0700 Subject: [PATCH 09/83] fix directory creation, directory listing --- weed/filer2/filer.go | 11 +++++++---- weed/filer2/memdb/memdb_store.go | 8 ++++++-- weed/server/filer_server_handlers_admin.go | 3 ++- weed/server/filer_server_handlers_read.go | 2 ++ 4 files changed, 17 insertions(+), 7 deletions(-) diff --git a/weed/filer2/filer.go b/weed/filer2/filer.go index 4ee6f4e50..467f846ed 100644 --- a/weed/filer2/filer.go +++ b/weed/filer2/filer.go @@ -9,6 +9,7 @@ import ( "time" "os" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" + "github.com/chrislusf/seaweedfs/weed/glog" ) type Filer struct { @@ -44,22 +45,23 @@ func (f *Filer) CreateEntry(entry *Entry) (error) { dirPath := "/" + filepath.Join(dirParts[:i]...) // fmt.Printf("%d directory: %+v\n", i, dirPath) - dirFound := false - // first check local cache dirEntry := f.cacheGetDirectory(dirPath) // not found, check the store directly if dirEntry == nil { + glog.V(4).Infof("find uncached directory: %s", dirPath) var dirFindErr error - dirFound, dirEntry, dirFindErr = f.FindEntry(FullPath(dirPath)) + _, dirEntry, dirFindErr = f.FindEntry(FullPath(dirPath)) if dirFindErr != nil { return fmt.Errorf("findDirectory %s: %v", dirPath, dirFindErr) } + }else{ + glog.V(4).Infof("found cached directory: %s", dirPath) } // no such existing directory - if !dirFound { + if dirEntry == nil { // create the directory now := time.Now() @@ -75,6 +77,7 @@ func (f *Filer) CreateEntry(entry *Entry) (error) { }, } + glog.V(2).Infof("create directory: %s", dirPath) mkdirErr := f.store.InsertEntry(dirEntry) if mkdirErr != nil { return fmt.Errorf("mkdir %s: %v", dirPath, mkdirErr) diff --git a/weed/filer2/memdb/memdb_store.go b/weed/filer2/memdb/memdb_store.go index 0276e9a7f..7e886f4ef 100644 --- a/weed/filer2/memdb/memdb_store.go +++ b/weed/filer2/memdb/memdb_store.go @@ -77,11 +77,13 @@ func (filer *MemDbStore) ListDirectoryEntries(fullpath filer2.FullPath, startFil } entry := item.(Entry).Entry // println("checking", entry.FullPath) + if entry.FullPath == fullpath { // skipping the current directory // println("skipping the folder", entry.FullPath) return true } + dir, name := entry.FullPath.DirAndName() if name == startFileName { if inclusive { @@ -90,11 +92,13 @@ func (filer *MemDbStore) ListDirectoryEntries(fullpath filer2.FullPath, startFil } return true } - if !strings.HasPrefix(dir, string(fullpath)) { - // println("directory is:", dir, "fullpath:", fullpath) + + // only iterate the same prefix + if !strings.HasPrefix(string(entry.FullPath), string(fullpath)) { // println("breaking from", entry.FullPath) return false } + if dir != string(fullpath) { // this could be items in deeper directories // println("skipping deeper folder", entry.FullPath) diff --git a/weed/server/filer_server_handlers_admin.go b/weed/server/filer_server_handlers_admin.go index 2c5c1bc21..06d041398 100644 --- a/weed/server/filer_server_handlers_admin.go +++ b/weed/server/filer_server_handlers_admin.go @@ -15,7 +15,7 @@ func (fs *FilerServer) registerHandler(w http.ResponseWriter, r *http.Request) { fileId := r.FormValue("fileId") fileSize, err := strconv.ParseUint(r.FormValue("fileSize"), 10, 64) if err != nil { - glog.V(4).Infof("register %s to %s parse fileSize %s: %v", fileId, path, r.FormValue("fileSize"), err) + glog.V(0).Infof("register %s to %s parse fileSize %s: %v", fileId, path, r.FormValue("fileSize"), err) writeJsonError(w, r, http.StatusInternalServerError, err) return } @@ -30,6 +30,7 @@ func (fs *FilerServer) registerHandler(w http.ResponseWriter, r *http.Request) { Mtime: time.Now().UnixNano(), }}, } + glog.V(2).Infof("register %s to %s parse fileSize %s", fileId, path, r.FormValue("fileSize")) err = fs.filer.CreateEntry(entry) if err != nil { glog.V(4).Infof("register %s to %s error: %v", fileId, path, err) diff --git a/weed/server/filer_server_handlers_read.go b/weed/server/filer_server_handlers_read.go index 3517299cf..52b7ba177 100644 --- a/weed/server/filer_server_handlers_read.go +++ b/weed/server/filer_server_handlers_read.go @@ -48,6 +48,8 @@ func (fs *FilerServer) listDirectoryHandler(w http.ResponseWriter, r *http.Reque lastFileName = entries[len(entries)-1].Name() } + glog.V(4).Infof("listDirectory %s, last file %s, limit %d: %d items", path, lastFileName, limit, len(entries)) + args := struct { Path string Entries interface{} From 793dd81ca2c81f4dab1cf65bb45ccb1e5726a54d Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sat, 19 May 2018 12:40:24 -0700 Subject: [PATCH 10/83] skip permission checking when creating dir or files --- weed/filer2/filer.go | 6 +++++- weed/filesys/dir.go | 1 + 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/weed/filer2/filer.go b/weed/filer2/filer.go index 467f846ed..2d5920cd8 100644 --- a/weed/filer2/filer.go +++ b/weed/filer2/filer.go @@ -56,7 +56,7 @@ func (f *Filer) CreateEntry(entry *Entry) (error) { if dirFindErr != nil { return fmt.Errorf("findDirectory %s: %v", dirPath, dirFindErr) } - }else{ + } else { glog.V(4).Infof("found cached directory: %s", dirPath) } @@ -98,9 +98,13 @@ func (f *Filer) CreateEntry(entry *Entry) (error) { return fmt.Errorf("parent folder not found: %v", entry.FullPath) } + /* if !hasWritePermission(lastDirectoryEntry, entry) { + glog.V(0).Infof("directory %s: %v, entry: uid=%d gid=%d", + lastDirectoryEntry.FullPath, lastDirectoryEntry.Attr, entry.Uid, entry.Gid) return fmt.Errorf("no write permission in folder %v", lastDirectoryEntry.FullPath) } + */ if err := f.store.InsertEntry(entry); err != nil { return fmt.Errorf("insert entry %s: %v", entry.FullPath, err) diff --git a/weed/filesys/dir.go b/weed/filesys/dir.go index f0191533d..4383fe62d 100644 --- a/weed/filesys/dir.go +++ b/weed/filesys/dir.go @@ -84,6 +84,7 @@ func (dir *Dir) Mkdir(ctx context.Context, req *fuse.MkdirRequest) (fs.Node, err glog.V(1).Infof("mkdir: %v", request) if _, err := client.CreateEntry(ctx, request); err != nil { + glog.V(0).Infof("mkdir %v: %v", request, err) return fmt.Errorf("make dir: %v", err) } From 7ca505294267b8f28c3b80839ab89fe99052b067 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sat, 19 May 2018 13:51:44 -0700 Subject: [PATCH 11/83] create files correctly! --- weed/filesys/dir.go | 3 +++ weed/filesys/file.go | 24 ++++++++++++++++++++++-- 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/weed/filesys/dir.go b/weed/filesys/dir.go index 4383fe62d..732912a99 100644 --- a/weed/filesys/dir.go +++ b/weed/filesys/dir.go @@ -21,6 +21,9 @@ type Dir struct { wfs *WFS } +var _ = fs.Node(&Dir{}) +var _ = fs.HandleReadDirAller(&Dir{}) + func (dir *Dir) Attr(context context.Context, attr *fuse.Attr) error { attr.Mode = os.ModeDir | 0777 return nil diff --git a/weed/filesys/file.go b/weed/filesys/file.go index 18fd05e1d..230ff272f 100644 --- a/weed/filesys/file.go +++ b/weed/filesys/file.go @@ -23,6 +23,7 @@ var _ = fs.HandleReadAller(&File{}) // var _ = fs.HandleReader(&File{}) var _ = fs.HandleFlusher(&File{}) var _ = fs.HandleWriter(&File{}) +var _ = fs.HandleReleaser(&File{}) type File struct { Chunks []*filer_pb.FileChunk @@ -49,6 +50,7 @@ func (file *File) Attr(context context.Context, attr *fuse.Attr) error { glog.V(1).Infof("read file size: %v", request) resp, err := client.GetFileAttributes(context, request) if err != nil { + glog.V(0).Infof("read file attributes %v: %v", request, err) return err } @@ -67,13 +69,17 @@ func (file *File) Attr(context context.Context, attr *fuse.Attr) error { attr.Mtime = time.Unix(attributes.Mtime, 0) attr.Gid = attributes.Gid attr.Uid = attributes.Uid + return nil } func (file *File) ReadAll(ctx context.Context) (content []byte, err error) { + // fmt.Printf("read all file %+v/%v\n", file.dir.Path, file.Name) + if len(file.Chunks) == 0 { + glog.V(0).Infof("empty file %v/%v", file.dir.Path, file.Name) return } @@ -109,7 +115,12 @@ func (file *File) Fsync(ctx context.Context, req *fuse.FsyncRequest) error { func (file *File) Flush(ctx context.Context, req *fuse.FlushRequest) error { // fflush works at file level // send the data to the OS - fmt.Printf("flush file %+v\n", req) + glog.V(3).Infof("file flush %v", req) + + if len(file.Chunks) == 0 { + glog.V(2).Infof("file flush skipping empty %v", req) + return nil + } err := file.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error { @@ -134,7 +145,7 @@ func (file *File) Flush(ctx context.Context, req *fuse.FlushRequest) error { func (file *File) Write(ctx context.Context, req *fuse.WriteRequest, resp *fuse.WriteResponse) error { // write the request to volume servers - fmt.Printf("write file %+v\n", req) + // fmt.Printf("write file %+v\n", req) var fileId, host string @@ -178,5 +189,14 @@ func (file *File) Write(ctx context.Context, req *fuse.WriteRequest, resp *fuse. Mtime: time.Now().UnixNano(), }) + resp.Size = int(uploadResult.Size) + + return nil +} + +func (file *File) Release(ctx context.Context, req *fuse.ReleaseRequest) error { + + // fmt.Printf("release file %+v\n", req) + return nil } From cc66e25cd28063ba8497ea3cf7eca084b60a661b Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sun, 20 May 2018 17:06:09 -0700 Subject: [PATCH 12/83] merge intervals --- weed/filer2/filechunks.go | 356 ++++++++++++++++++++++++++++++- weed/filer2/filechunks_test.go | 112 ++++++++++ weed/server/filer_grpc_server.go | 5 +- 3 files changed, 460 insertions(+), 13 deletions(-) create mode 100644 weed/filer2/filechunks_test.go diff --git a/weed/filer2/filechunks.go b/weed/filer2/filechunks.go index 9ddbe236e..257aba548 100644 --- a/weed/filer2/filechunks.go +++ b/weed/filer2/filechunks.go @@ -1,10 +1,13 @@ package filer2 -import "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" +import ( + "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" + "sort" + "log" + "math" +) -type Chunks []*filer_pb.FileChunk - -func (chunks Chunks) TotalSize() (size uint64) { +func TotalSize(chunks []*filer_pb.FileChunk) (size uint64) { for _, c := range chunks { t := uint64(c.Offset + int64(c.Size)) if size < t { @@ -14,12 +17,345 @@ func (chunks Chunks) TotalSize() (size uint64) { return } -func (chunks Chunks) Len() int { - return len(chunks) +func CompactFileChunks(chunks []*filer_pb.FileChunk) (compacted, garbage []*filer_pb.FileChunk) { + return } -func (chunks Chunks) Swap(i, j int) { - chunks[i], chunks[j] = chunks[j], chunks[i] + +func mergeToVisibleIntervals(visibles []*visibleInterval, chunk *filer_pb.FileChunk) (merged []*visibleInterval) { + if len(visibles) == 0 { + return []*visibleInterval{newVisibleInterval(chunk.Offset, chunk.Offset+int64(chunk.Size), chunk.FileId, chunk.Mtime)} + } + + log.Printf("merge chunk %+v => %d", chunk, len(visibles)) + for _, v := range visibles { + log.Printf("=> %+v", v) + } + + var nonOverlappingStop int + + // find merge candidates + var mergeCandidates []int + for t := len(visibles) - 1; t >= 0; t-- { + if visibles[t].stop > chunk.Offset { + mergeCandidates = append(mergeCandidates, t) + } else { + nonOverlappingStop = t + break + } + } + log.Printf("merged candidates: %+v, starting from %d", mergeCandidates, nonOverlappingStop) + + if len(mergeCandidates) == 0 { + merged = append(visibles, newVisibleInterval( + chunk.Offset, + chunk.Offset+int64(chunk.Size), + chunk.FileId, + chunk.Mtime, + )) + return + } + + // reverse merge candidates + i, j := 0, len(mergeCandidates)-1 + for i < j { + mergeCandidates[i], mergeCandidates[j] = mergeCandidates[j], mergeCandidates[i] + i++ + j-- + } + log.Printf("reversed merged candidates: %+v", mergeCandidates) + + // add chunk into a possibly connected intervals + var overlappingIntevals []*visibleInterval + for i = 0; i < len(mergeCandidates); i++ { + interval := visibles[mergeCandidates[i]] + if interval.modifiedTime >= chunk.Mtime { + log.Printf("overlappingIntevals add existing interval: [%d,%d)", interval.start, interval.stop) + overlappingIntevals = append(overlappingIntevals, interval) + } else { + start := max(interval.start, chunk.Offset) + stop := min(interval.stop, chunk.Offset+int64(chunk.Size)) + if interval.start <= chunk.Offset { + if interval.start < start { + log.Printf("overlappingIntevals add 1: [%d,%d)", interval.start, start) + overlappingIntevals = append(overlappingIntevals, newVisibleInterval( + interval.start, + start, + interval.fileId, + interval.modifiedTime, + )) + } + log.Printf("overlappingIntevals add 2: [%d,%d)", start, stop) + overlappingIntevals = append(overlappingIntevals, newVisibleInterval( + start, + stop, + chunk.FileId, + chunk.Mtime, + )) + if interval.stop < stop { + log.Printf("overlappingIntevals add 3: [%d,%d)", interval.stop, stop) + overlappingIntevals = append(overlappingIntevals, newVisibleInterval( + interval.stop, + stop, + interval.fileId, + interval.modifiedTime, + )) + } + } + } + } + logPrintf("overlappingIntevals", overlappingIntevals) + + // merge connected intervals + merged = visibles[:nonOverlappingStop] + var lastInterval *visibleInterval + var prevIntervalIndex int + for i, interval := range overlappingIntevals { + if i == 0 { + prevIntervalIndex = i + continue + } + if overlappingIntevals[prevIntervalIndex].fileId != interval.fileId { + merged = append(merged, newVisibleInterval( + overlappingIntevals[prevIntervalIndex].start, + interval.start, + overlappingIntevals[prevIntervalIndex].fileId, + overlappingIntevals[prevIntervalIndex].modifiedTime, + )) + prevIntervalIndex = i + } + } + + if lastInterval != nil { + merged = append(merged, newVisibleInterval( + overlappingIntevals[prevIntervalIndex].start, + lastInterval.start, + overlappingIntevals[prevIntervalIndex].fileId, + overlappingIntevals[prevIntervalIndex].modifiedTime, + )) + } + + logPrintf("merged", merged) + + return +} + +func logPrintf(name string, visibles []*visibleInterval) { + log.Printf("%s len %d", name, len(visibles)) + for _, v := range visibles { + log.Printf("%s: => %+v", name, v) + } } -func (chunks Chunks) Less(i, j int) bool { - return chunks[i].Offset < chunks[j].Offset + +func nonOverlappingVisibleIntervals(chunks []*filer_pb.FileChunk) (visibles []*visibleInterval) { + + sort.Slice(chunks, func(i, j int) bool { + if chunks[i].Offset < chunks[j].Offset { + return true + } + if chunks[i].Offset == chunks[j].Offset { + return chunks[i].Mtime < chunks[j].Mtime + } + return false + }) + + if len(chunks) == 0 { + return + } + + var parallelIntervals, intervals []*visibleInterval + var minStopInterval, upToDateInterval *visibleInterval + watermarkStart := chunks[0].Offset + for _, chunk := range chunks { + log.Printf("checking chunk: [%d,%d)", chunk.Offset, chunk.Offset+int64(chunk.Size)) + logPrintf("parallelIntervals", parallelIntervals) + for len(parallelIntervals) > 0 && watermarkStart < chunk.Offset { + logPrintf("parallelIntervals loop 1", parallelIntervals) + logPrintf("parallelIntervals loop 1 intervals", intervals) + minStopInterval, upToDateInterval = findMinStopInterval(parallelIntervals) + nextStop := min(minStopInterval.stop, chunk.Offset) + intervals = append(intervals, newVisibleInterval( + max(watermarkStart, minStopInterval.start), + nextStop, + upToDateInterval.fileId, + upToDateInterval.modifiedTime, + )) + watermarkStart = nextStop + logPrintf("parallelIntervals loop intervals =>", intervals) + + // remove processed intervals, possibly multiple + var remaining []*visibleInterval + for _, interval := range parallelIntervals { + if interval.stop != watermarkStart { + remaining = append(remaining, newVisibleInterval( + interval.start, + interval.stop, + interval.fileId, + interval.modifiedTime, + )) + } + } + parallelIntervals = remaining + logPrintf("parallelIntervals loop 2", parallelIntervals) + logPrintf("parallelIntervals loop 2 intervals", intervals) + } + parallelIntervals = append(parallelIntervals, newVisibleInterval( + chunk.Offset, + chunk.Offset+int64(chunk.Size), + chunk.FileId, + chunk.Mtime, + )) + } + + logPrintf("parallelIntervals loop 3", parallelIntervals) + logPrintf("parallelIntervals loop 3 intervals", intervals) + for len(parallelIntervals) > 0 { + minStopInterval, upToDateInterval = findMinStopInterval(parallelIntervals) + intervals = append(intervals, newVisibleInterval( + max(watermarkStart, minStopInterval.start), + minStopInterval.stop, + upToDateInterval.fileId, + upToDateInterval.modifiedTime, + )) + watermarkStart = minStopInterval.stop + + // remove processed intervals, possibly multiple + var remaining []*visibleInterval + for _, interval := range parallelIntervals { + if interval.stop != watermarkStart { + remaining = append(remaining, newVisibleInterval( + interval.start, + interval.stop, + interval.fileId, + interval.modifiedTime, + )) + } + } + parallelIntervals = remaining + } + logPrintf("parallelIntervals loop 4", parallelIntervals) + logPrintf("intervals", intervals) + + // merge connected intervals, now the intervals are non-intersecting + var lastInterval *visibleInterval + var prevIntervalIndex int + for i, interval := range intervals { + if i == 0 { + prevIntervalIndex = i + continue + } + if intervals[i-1].fileId != interval.fileId || + intervals[i-1].stop < intervals[i].start { + visibles = append(visibles, newVisibleInterval( + intervals[prevIntervalIndex].start, + intervals[i-1].stop, + intervals[prevIntervalIndex].fileId, + intervals[prevIntervalIndex].modifiedTime, + )) + prevIntervalIndex = i + } + lastInterval = intervals[i] + logPrintf("intervals loop 1 visibles", visibles) + } + + if lastInterval != nil { + visibles = append(visibles, newVisibleInterval( + intervals[prevIntervalIndex].start, + lastInterval.stop, + intervals[prevIntervalIndex].fileId, + intervals[prevIntervalIndex].modifiedTime, + )) + } + + logPrintf("visibles", visibles) + + return +} + +func findMinStopInterval(intervals []*visibleInterval) (minStopInterval, upToDateInterval *visibleInterval) { + var latestMtime int64 + latestIntervalIndex := 0 + minStop := int64(math.MaxInt64) + minIntervalIndex := 0 + for i, interval := range intervals { + if minStop > interval.stop { + minIntervalIndex = i + minStop = interval.stop + } + if latestMtime < interval.modifiedTime { + latestMtime = interval.modifiedTime + latestIntervalIndex = i + } + } + minStopInterval = intervals[minIntervalIndex] + upToDateInterval = intervals[latestIntervalIndex] + return +} + +func nonOverlappingVisibleIntervals0(chunks []*filer_pb.FileChunk) (visibles []*visibleInterval) { + + sort.Slice(chunks, func(i, j int) bool { + if chunks[i].Offset < chunks[j].Offset { + return true + } + if chunks[i].Offset == chunks[j].Offset { + return chunks[i].Mtime < chunks[j].Mtime + } + return false + }) + + for _, c := range chunks { + visibles = mergeToVisibleIntervals(visibles, c) + } + + return +} + +// find non-overlapping visible intervals +// visible interval map to one file chunk + +type visibleInterval struct { + start int64 + stop int64 + modifiedTime int64 + fileId string +} + +func newVisibleInterval(start, stop int64, fileId string, modifiedTime int64) *visibleInterval { + return &visibleInterval{start: start, stop: stop, fileId: fileId, modifiedTime: modifiedTime} +} + +type stackOfChunkIds struct { + ids []int +} + +func (s *stackOfChunkIds) isEmpty() bool { + return len(s.ids) == 0 +} + +func (s *stackOfChunkIds) pop() int { + t := s.ids[len(s.ids)-1] + s.ids = s.ids[:len(s.ids)-1] + return t +} + +func (s *stackOfChunkIds) push(x int) { + s.ids = append(s.ids, x) +} + +func (s *stackOfChunkIds) peek() int { + return s.ids[len(s.ids)-1] +} + +func min(x, y int64) int64 { + if x <= y { + return x + } + return y +} + +func max(x, y int64) int64 { + if x > y { + return x + } + return y } diff --git a/weed/filer2/filechunks_test.go b/weed/filer2/filechunks_test.go new file mode 100644 index 000000000..e5cb8810f --- /dev/null +++ b/weed/filer2/filechunks_test.go @@ -0,0 +1,112 @@ +package filer2 + +import ( + "testing" + "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" + "log" +) + +func TestIntervalMerging(t *testing.T) { + + testcases := []struct { + Chunks []*filer_pb.FileChunk + Expected []*visibleInterval + }{ + // case 0: normal + { + Chunks: []*filer_pb.FileChunk{ + {Offset: 0, Size: 100, FileId: "abc", Mtime: 123}, + {Offset: 100, Size: 100, FileId: "asdf", Mtime: 134}, + {Offset: 200, Size: 100, FileId: "fsad", Mtime: 353}, + }, + Expected: []*visibleInterval{ + {start: 0, stop: 100, fileId: "abc"}, + {start: 100, stop: 200, fileId: "asdf"}, + {start: 200, stop: 300, fileId: "fsad"}, + }, + }, + // case 1: updates overwrite full chunks + { + Chunks: []*filer_pb.FileChunk{ + {Offset: 0, Size: 100, FileId: "abc", Mtime: 123}, + {Offset: 0, Size: 200, FileId: "asdf", Mtime: 134}, + }, + Expected: []*visibleInterval{ + {start: 0, stop: 200, fileId: "asdf"}, + }, + }, + // case 2: updates overwrite part of previous chunks + { + Chunks: []*filer_pb.FileChunk{ + {Offset: 0, Size: 100, FileId: "abc", Mtime: 123}, + {Offset: 0, Size: 50, FileId: "asdf", Mtime: 134}, + }, + Expected: []*visibleInterval{ + {start: 0, stop: 50, fileId: "asdf"}, + {start: 50, stop: 100, fileId: "abc"}, + }, + }, + // case 3: updates overwrite full chunks + { + Chunks: []*filer_pb.FileChunk{ + {Offset: 0, Size: 100, FileId: "abc", Mtime: 123}, + {Offset: 0, Size: 200, FileId: "asdf", Mtime: 134}, + {Offset: 50, Size: 250, FileId: "xxxx", Mtime: 154}, + }, + Expected: []*visibleInterval{ + {start: 0, stop: 50, fileId: "asdf"}, + {start: 50, stop: 300, fileId: "xxxx"}, + }, + }, + // case 4: updates far away from prev chunks + { + Chunks: []*filer_pb.FileChunk{ + {Offset: 0, Size: 100, FileId: "abc", Mtime: 123}, + {Offset: 0, Size: 200, FileId: "asdf", Mtime: 134}, + {Offset: 250, Size: 250, FileId: "xxxx", Mtime: 154}, + }, + Expected: []*visibleInterval{ + {start: 0, stop: 200, fileId: "asdf"}, + {start: 250, stop: 500, fileId: "xxxx"}, + }, + }, + // case 5: updates overwrite full chunks + { + Chunks: []*filer_pb.FileChunk{ + {Offset: 0, Size: 100, FileId: "abc", Mtime: 123}, + {Offset: 0, Size: 200, FileId: "asdf", Mtime: 184}, + {Offset: 70, Size: 150, FileId: "abc", Mtime: 143}, + {Offset: 80, Size: 100, FileId: "xxxx", Mtime: 134}, + }, + Expected: []*visibleInterval{ + {start: 0, stop: 200, fileId: "asdf"}, + {start: 200, stop: 220, fileId: "abc"}, + }, + }, + } + + for i, testcase := range testcases { + log.Printf("++++++++++ merged test case %d ++++++++++++++++++++", i) + intervals := nonOverlappingVisibleIntervals(testcase.Chunks) + for x, interval := range intervals { + log.Printf("test case %d, interval %d, start=%d, stop=%d, fileId=%s", + i, x, interval.start, interval.stop, interval.fileId) + if interval.start != testcase.Expected[x].start { + t.Fatalf("failed on test case %d, interval %d, start %d, expect %d", + i, x, interval.start, testcase.Expected[x].start) + } + if interval.stop != testcase.Expected[x].stop { + t.Fatalf("failed on test case %d, interval %d, stop %d, expect %d", + i, x, interval.stop, testcase.Expected[x].stop) + } + if interval.fileId != testcase.Expected[x].fileId { + t.Fatalf("failed on test case %d, interval %d, chunkId %s, expect %s", + i, x, interval.fileId, testcase.Expected[x].fileId) + } + } + if len(intervals) != len(testcase.Expected) { + t.Fatalf("failed to compact test case %d, len %d expected %d", i, len(intervals), len(testcase.Expected)) + } + } + +} diff --git a/weed/server/filer_grpc_server.go b/weed/server/filer_grpc_server.go index 46a534ffa..6cc5021d6 100644 --- a/weed/server/filer_grpc_server.go +++ b/weed/server/filer_grpc_server.go @@ -42,13 +42,12 @@ func (fs *FilerServer) ListEntries(ctx context.Context, req *filer_pb.ListEntrie resp := &filer_pb.ListEntriesResponse{} for _, entry := range entries { - glog.V(0).Infof("%s attr=%v size=%d", entry.Name(), entry.Attr, filer2.Chunks(entry.Chunks).TotalSize()) resp.Entries = append(resp.Entries, &filer_pb.Entry{ Name: entry.Name(), IsDirectory: entry.IsDirectory(), Chunks: entry.Chunks, Attributes: &filer_pb.FuseAttributes{ - FileSize: filer2.Chunks(entry.Chunks).TotalSize(), + FileSize: filer2.TotalSize(entry.Chunks), Mtime: entry.Mtime.Unix(), Gid: entry.Gid, Uid: entry.Uid, @@ -71,7 +70,7 @@ func (fs *FilerServer) GetFileAttributes(ctx context.Context, req *filer_pb.GetF if !found { attributes.FileSize = 0 } else { - attributes.FileSize = filer2.Chunks(entry.Chunks).TotalSize() + attributes.FileSize = filer2.TotalSize(entry.Chunks) attributes.FileMode = uint32(entry.Mode) attributes.Uid = entry.Uid attributes.Gid = entry.Gid From cd47528a757dd38d68c9137bd4eb97f4b9f4e305 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sun, 20 May 2018 17:08:54 -0700 Subject: [PATCH 13/83] remove unused code --- weed/filer2/filechunks.go | 163 +-------------------------------- weed/filer2/filechunks_test.go | 3 +- 2 files changed, 6 insertions(+), 160 deletions(-) diff --git a/weed/filer2/filechunks.go b/weed/filer2/filechunks.go index 257aba548..88a2660cb 100644 --- a/weed/filer2/filechunks.go +++ b/weed/filer2/filechunks.go @@ -1,10 +1,11 @@ package filer2 import ( - "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "sort" "log" "math" + + "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" ) func TotalSize(chunks []*filer_pb.FileChunk) (size uint64) { @@ -21,125 +22,10 @@ func CompactFileChunks(chunks []*filer_pb.FileChunk) (compacted, garbage []*file return } -func mergeToVisibleIntervals(visibles []*visibleInterval, chunk *filer_pb.FileChunk) (merged []*visibleInterval) { - if len(visibles) == 0 { - return []*visibleInterval{newVisibleInterval(chunk.Offset, chunk.Offset+int64(chunk.Size), chunk.FileId, chunk.Mtime)} - } - - log.Printf("merge chunk %+v => %d", chunk, len(visibles)) - for _, v := range visibles { - log.Printf("=> %+v", v) - } - - var nonOverlappingStop int - - // find merge candidates - var mergeCandidates []int - for t := len(visibles) - 1; t >= 0; t-- { - if visibles[t].stop > chunk.Offset { - mergeCandidates = append(mergeCandidates, t) - } else { - nonOverlappingStop = t - break - } - } - log.Printf("merged candidates: %+v, starting from %d", mergeCandidates, nonOverlappingStop) - - if len(mergeCandidates) == 0 { - merged = append(visibles, newVisibleInterval( - chunk.Offset, - chunk.Offset+int64(chunk.Size), - chunk.FileId, - chunk.Mtime, - )) - return - } - - // reverse merge candidates - i, j := 0, len(mergeCandidates)-1 - for i < j { - mergeCandidates[i], mergeCandidates[j] = mergeCandidates[j], mergeCandidates[i] - i++ - j-- - } - log.Printf("reversed merged candidates: %+v", mergeCandidates) - - // add chunk into a possibly connected intervals - var overlappingIntevals []*visibleInterval - for i = 0; i < len(mergeCandidates); i++ { - interval := visibles[mergeCandidates[i]] - if interval.modifiedTime >= chunk.Mtime { - log.Printf("overlappingIntevals add existing interval: [%d,%d)", interval.start, interval.stop) - overlappingIntevals = append(overlappingIntevals, interval) - } else { - start := max(interval.start, chunk.Offset) - stop := min(interval.stop, chunk.Offset+int64(chunk.Size)) - if interval.start <= chunk.Offset { - if interval.start < start { - log.Printf("overlappingIntevals add 1: [%d,%d)", interval.start, start) - overlappingIntevals = append(overlappingIntevals, newVisibleInterval( - interval.start, - start, - interval.fileId, - interval.modifiedTime, - )) - } - log.Printf("overlappingIntevals add 2: [%d,%d)", start, stop) - overlappingIntevals = append(overlappingIntevals, newVisibleInterval( - start, - stop, - chunk.FileId, - chunk.Mtime, - )) - if interval.stop < stop { - log.Printf("overlappingIntevals add 3: [%d,%d)", interval.stop, stop) - overlappingIntevals = append(overlappingIntevals, newVisibleInterval( - interval.stop, - stop, - interval.fileId, - interval.modifiedTime, - )) - } - } - } - } - logPrintf("overlappingIntevals", overlappingIntevals) - - // merge connected intervals - merged = visibles[:nonOverlappingStop] - var lastInterval *visibleInterval - var prevIntervalIndex int - for i, interval := range overlappingIntevals { - if i == 0 { - prevIntervalIndex = i - continue - } - if overlappingIntevals[prevIntervalIndex].fileId != interval.fileId { - merged = append(merged, newVisibleInterval( - overlappingIntevals[prevIntervalIndex].start, - interval.start, - overlappingIntevals[prevIntervalIndex].fileId, - overlappingIntevals[prevIntervalIndex].modifiedTime, - )) - prevIntervalIndex = i - } - } - - if lastInterval != nil { - merged = append(merged, newVisibleInterval( - overlappingIntevals[prevIntervalIndex].start, - lastInterval.start, - overlappingIntevals[prevIntervalIndex].fileId, - overlappingIntevals[prevIntervalIndex].modifiedTime, - )) - } - - logPrintf("merged", merged) +func logPrintf(name string, visibles []*visibleInterval) { return -} -func logPrintf(name string, visibles []*visibleInterval) { log.Printf("%s len %d", name, len(visibles)) for _, v := range visibles { log.Printf("%s: => %+v", name, v) @@ -166,7 +52,7 @@ func nonOverlappingVisibleIntervals(chunks []*filer_pb.FileChunk) (visibles []*v var minStopInterval, upToDateInterval *visibleInterval watermarkStart := chunks[0].Offset for _, chunk := range chunks { - log.Printf("checking chunk: [%d,%d)", chunk.Offset, chunk.Offset+int64(chunk.Size)) + // log.Printf("checking chunk: [%d,%d)", chunk.Offset, chunk.Offset+int64(chunk.Size)) logPrintf("parallelIntervals", parallelIntervals) for len(parallelIntervals) > 0 && watermarkStart < chunk.Offset { logPrintf("parallelIntervals loop 1", parallelIntervals) @@ -291,25 +177,6 @@ func findMinStopInterval(intervals []*visibleInterval) (minStopInterval, upToDat return } -func nonOverlappingVisibleIntervals0(chunks []*filer_pb.FileChunk) (visibles []*visibleInterval) { - - sort.Slice(chunks, func(i, j int) bool { - if chunks[i].Offset < chunks[j].Offset { - return true - } - if chunks[i].Offset == chunks[j].Offset { - return chunks[i].Mtime < chunks[j].Mtime - } - return false - }) - - for _, c := range chunks { - visibles = mergeToVisibleIntervals(visibles, c) - } - - return -} - // find non-overlapping visible intervals // visible interval map to one file chunk @@ -324,28 +191,6 @@ func newVisibleInterval(start, stop int64, fileId string, modifiedTime int64) *v return &visibleInterval{start: start, stop: stop, fileId: fileId, modifiedTime: modifiedTime} } -type stackOfChunkIds struct { - ids []int -} - -func (s *stackOfChunkIds) isEmpty() bool { - return len(s.ids) == 0 -} - -func (s *stackOfChunkIds) pop() int { - t := s.ids[len(s.ids)-1] - s.ids = s.ids[:len(s.ids)-1] - return t -} - -func (s *stackOfChunkIds) push(x int) { - s.ids = append(s.ids, x) -} - -func (s *stackOfChunkIds) peek() int { - return s.ids[len(s.ids)-1] -} - func min(x, y int64) int64 { if x <= y { return x diff --git a/weed/filer2/filechunks_test.go b/weed/filer2/filechunks_test.go index e5cb8810f..a1614641a 100644 --- a/weed/filer2/filechunks_test.go +++ b/weed/filer2/filechunks_test.go @@ -2,8 +2,9 @@ package filer2 import ( "testing" - "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "log" + + "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" ) func TestIntervalMerging(t *testing.T) { From f07482382b067b84e917e4d1c3581fe1373957de Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Mon, 21 May 2018 00:00:28 -0700 Subject: [PATCH 14/83] able to update file content having some issue when vi reports file changed. --- weed/filer2/embedded/embedded_store.go | 2 +- weed/filer2/filechunks.go | 52 +++++---- weed/filer2/filechunks_test.go | 33 ++++++ weed/filer2/filer.go | 4 +- weed/filer2/filer_structure.go | 2 +- weed/filer2/memdb/memdb_store.go | 5 +- weed/filesys/dir.go | 2 +- weed/filesys/file.go | 42 +++++-- weed/pb/filer.proto | 6 +- weed/pb/filer_pb/filer.pb.go | 154 ++++++++++++------------- weed/server/filer_grpc_server.go | 31 ++++- 11 files changed, 209 insertions(+), 124 deletions(-) diff --git a/weed/filer2/embedded/embedded_store.go b/weed/filer2/embedded/embedded_store.go index f871f18e2..d0cfac3f9 100644 --- a/weed/filer2/embedded/embedded_store.go +++ b/weed/filer2/embedded/embedded_store.go @@ -26,7 +26,7 @@ func (filer *EmbeddedStore) AddDirectoryLink(directory *filer2.Entry, delta int3 return nil } -func (filer *EmbeddedStore) AppendFileChunk(fullpath filer2.FullPath, fileChunks []*filer_pb.FileChunk) (err error) { +func (filer *EmbeddedStore) SetFileChunks(fullpath filer2.FullPath, fileChunks []*filer_pb.FileChunk) (err error) { return nil } diff --git a/weed/filer2/filechunks.go b/weed/filer2/filechunks.go index 88a2660cb..714bdab9c 100644 --- a/weed/filer2/filechunks.go +++ b/weed/filer2/filechunks.go @@ -19,12 +19,27 @@ func TotalSize(chunks []*filer_pb.FileChunk) (size uint64) { } func CompactFileChunks(chunks []*filer_pb.FileChunk) (compacted, garbage []*filer_pb.FileChunk) { + + visibles := nonOverlappingVisibleIntervals(chunks) + + fileIds := make(map[string]bool) + for _, interval := range visibles { + fileIds[interval.fileId] = true + } + for _, chunk := range chunks { + if found := fileIds[chunk.FileId]; found { + compacted = append(compacted, chunk) + } else { + garbage = append(garbage, chunk) + } + } + return } func logPrintf(name string, visibles []*visibleInterval) { - return + // return log.Printf("%s len %d", name, len(visibles)) for _, v := range visibles { @@ -52,7 +67,7 @@ func nonOverlappingVisibleIntervals(chunks []*filer_pb.FileChunk) (visibles []*v var minStopInterval, upToDateInterval *visibleInterval watermarkStart := chunks[0].Offset for _, chunk := range chunks { - // log.Printf("checking chunk: [%d,%d)", chunk.Offset, chunk.Offset+int64(chunk.Size)) + log.Printf("checking chunk: [%d,%d)", chunk.Offset, chunk.Offset+int64(chunk.Size)) logPrintf("parallelIntervals", parallelIntervals) for len(parallelIntervals) > 0 && watermarkStart < chunk.Offset { logPrintf("parallelIntervals loop 1", parallelIntervals) @@ -72,12 +87,7 @@ func nonOverlappingVisibleIntervals(chunks []*filer_pb.FileChunk) (visibles []*v var remaining []*visibleInterval for _, interval := range parallelIntervals { if interval.stop != watermarkStart { - remaining = append(remaining, newVisibleInterval( - interval.start, - interval.stop, - interval.fileId, - interval.modifiedTime, - )) + remaining = append(remaining, interval) } } parallelIntervals = remaining @@ -108,12 +118,7 @@ func nonOverlappingVisibleIntervals(chunks []*filer_pb.FileChunk) (visibles []*v var remaining []*visibleInterval for _, interval := range parallelIntervals { if interval.stop != watermarkStart { - remaining = append(remaining, newVisibleInterval( - interval.start, - interval.stop, - interval.fileId, - interval.modifiedTime, - )) + remaining = append(remaining, interval) } } parallelIntervals = remaining @@ -122,11 +127,12 @@ func nonOverlappingVisibleIntervals(chunks []*filer_pb.FileChunk) (visibles []*v logPrintf("intervals", intervals) // merge connected intervals, now the intervals are non-intersecting - var lastInterval *visibleInterval + var lastIntervalIndex int var prevIntervalIndex int for i, interval := range intervals { if i == 0 { prevIntervalIndex = i + lastIntervalIndex = i continue } if intervals[i-1].fileId != interval.fileId || @@ -139,18 +145,16 @@ func nonOverlappingVisibleIntervals(chunks []*filer_pb.FileChunk) (visibles []*v )) prevIntervalIndex = i } - lastInterval = intervals[i] + lastIntervalIndex = i logPrintf("intervals loop 1 visibles", visibles) } - if lastInterval != nil { - visibles = append(visibles, newVisibleInterval( - intervals[prevIntervalIndex].start, - lastInterval.stop, - intervals[prevIntervalIndex].fileId, - intervals[prevIntervalIndex].modifiedTime, - )) - } + visibles = append(visibles, newVisibleInterval( + intervals[prevIntervalIndex].start, + intervals[lastIntervalIndex].stop, + intervals[prevIntervalIndex].fileId, + intervals[prevIntervalIndex].modifiedTime, + )) logPrintf("visibles", visibles) diff --git a/weed/filer2/filechunks_test.go b/weed/filer2/filechunks_test.go index a1614641a..b87b61d3b 100644 --- a/weed/filer2/filechunks_test.go +++ b/weed/filer2/filechunks_test.go @@ -7,6 +7,28 @@ import ( "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" ) +func TestCompactFileChunks(t *testing.T) { + chunks := []*filer_pb.FileChunk{ + {Offset:10, Size:100, FileId:"abc", Mtime:50}, + {Offset:100, Size:100, FileId:"def", Mtime:100}, + {Offset:200, Size:100, FileId:"ghi", Mtime:200}, + {Offset:110, Size:200, FileId:"jkl", Mtime:300}, + } + + compacted, garbarge := CompactFileChunks(chunks) + + log.Printf("Compacted: %+v", compacted) + log.Printf("Garbage : %+v", garbarge) + + if len(compacted) != 3 { + t.Fatalf("unexpected compacted: %d", len(compacted)) + } + if len(garbarge) != 1 { + t.Fatalf("unexpected garbarge: %d", len(garbarge)) + } + +} + func TestIntervalMerging(t *testing.T) { testcases := []struct { @@ -84,6 +106,17 @@ func TestIntervalMerging(t *testing.T) { {start: 200, stop: 220, fileId: "abc"}, }, }, + // case 6: same updates + { + Chunks: []*filer_pb.FileChunk{ + {Offset: 0, Size: 100, FileId: "abc", Mtime: 123}, + {Offset: 0, Size: 100, FileId: "abc", Mtime: 123}, + {Offset: 0, Size: 100, FileId: "abc", Mtime: 123}, + }, + Expected: []*visibleInterval{ + {start: 0, stop: 100, fileId: "abc"}, + }, + }, } for i, testcase := range testcases { diff --git a/weed/filer2/filer.go b/weed/filer2/filer.go index 2d5920cd8..fd27760bf 100644 --- a/weed/filer2/filer.go +++ b/weed/filer2/filer.go @@ -113,8 +113,8 @@ func (f *Filer) CreateEntry(entry *Entry) (error) { return nil } -func (f *Filer) AppendFileChunk(p FullPath, chunks []*filer_pb.FileChunk) (err error) { - return f.store.AppendFileChunk(p, chunks) +func (f *Filer) SetFileChunks(p FullPath, chunks []*filer_pb.FileChunk) (err error) { + return f.store.SetFileChunks(p, chunks) } func (f *Filer) FindEntry(p FullPath) (found bool, entry *Entry, err error) { diff --git a/weed/filer2/filer_structure.go b/weed/filer2/filer_structure.go index 1d2da752d..51de09718 100644 --- a/weed/filer2/filer_structure.go +++ b/weed/filer2/filer_structure.go @@ -68,7 +68,7 @@ var ErrNotFound = errors.New("filer: no entry is found in filer store") type FilerStore interface { InsertEntry(*Entry) (error) - AppendFileChunk(FullPath, []*filer_pb.FileChunk) (err error) + SetFileChunks(FullPath, []*filer_pb.FileChunk) (err error) FindEntry(FullPath) (found bool, entry *Entry, err error) DeleteEntry(FullPath) (fileEntry *Entry, err error) diff --git a/weed/filer2/memdb/memdb_store.go b/weed/filer2/memdb/memdb_store.go index 7e886f4ef..1e888cc45 100644 --- a/weed/filer2/memdb/memdb_store.go +++ b/weed/filer2/memdb/memdb_store.go @@ -33,14 +33,13 @@ func (filer *MemDbStore) InsertEntry(entry *filer2.Entry) (err error) { return nil } -func (filer *MemDbStore) AppendFileChunk(fullpath filer2.FullPath, fileChunks []*filer_pb.FileChunk) (err error) { +func (filer *MemDbStore) SetFileChunks(fullpath filer2.FullPath, fileChunks []*filer_pb.FileChunk) (err error) { found, entry, err := filer.FindEntry(fullpath) if !found { return fmt.Errorf("No such file: %s", fullpath) } - entry.Chunks = append(entry.Chunks, fileChunks...) + entry.Chunks = fileChunks entry.Mtime = time.Now() - println("appending to entry", entry.Name(), len(entry.Chunks)) filer.tree.ReplaceOrInsert(Entry{entry}) return nil } diff --git a/weed/filesys/dir.go b/weed/filesys/dir.go index 732912a99..50b37f883 100644 --- a/weed/filesys/dir.go +++ b/weed/filesys/dir.go @@ -169,7 +169,7 @@ func (dir *Dir) ReadDirAll(ctx context.Context) (ret []fuse.Dirent, err error) { } else { dirent := fuse.Dirent{Name: entry.Name, Type: fuse.DT_File} ret = append(ret, dirent) - dir.wfs.listDirectoryEntriesCache.Set(dir.Path+"/"+entry.Name, entry.Attributes, 3*time.Second) + dir.wfs.listDirectoryEntriesCache.Set(dir.Path+"/"+entry.Name, entry.Attributes, 300*time.Millisecond) } } diff --git a/weed/filesys/file.go b/weed/filesys/file.go index 230ff272f..d4a9c8b05 100644 --- a/weed/filesys/file.go +++ b/weed/filesys/file.go @@ -13,6 +13,7 @@ import ( "time" "bytes" "github.com/chrislusf/seaweedfs/weed/operation" + "github.com/chrislusf/seaweedfs/weed/filer2" ) var _ = fs.Node(&File{}) @@ -24,6 +25,7 @@ var _ = fs.HandleReadAller(&File{}) var _ = fs.HandleFlusher(&File{}) var _ = fs.HandleWriter(&File{}) var _ = fs.HandleReleaser(&File{}) +var _ = fs.NodeSetattrer(&File{}) type File struct { Chunks []*filer_pb.FileChunk @@ -74,9 +76,32 @@ func (file *File) Attr(context context.Context, attr *fuse.Attr) error { } +func (file *File) xOpen(ctx context.Context, req *fuse.OpenRequest, resp *fuse.OpenResponse) (fs.Handle, error) { + fullPath := filepath.Join(file.dir.Path, file.Name) + + fmt.Printf("Open %v %+v\n", fullPath, req) + + return file, nil + +} + +func (file *File) Setattr(ctx context.Context, req *fuse.SetattrRequest, resp *fuse.SetattrResponse) error { + fullPath := filepath.Join(file.dir.Path, file.Name) + + fmt.Printf("Setattr %v %+v\n", fullPath, req) + if req.Valid.Size() && req.Size == 0 { + fmt.Printf("truncate %v \n", fullPath) + file.Chunks = nil + resp.Attr.Size = 0 + } + + return nil + +} + func (file *File) ReadAll(ctx context.Context) (content []byte, err error) { - // fmt.Printf("read all file %+v/%v\n", file.dir.Path, file.Name) + fmt.Printf("read all file %+v/%v\n", file.dir.Path, file.Name) if len(file.Chunks) == 0 { glog.V(0).Infof("empty file %v/%v", file.dir.Path, file.Name) @@ -86,11 +111,14 @@ func (file *File) ReadAll(ctx context.Context) (content []byte, err error) { err = file.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error { // FIXME: need to either use Read() or implement differently + chunks, _ := filer2.CompactFileChunks(file.Chunks) + glog.V(1).Infof("read file %v/%v %d/%d chunks", file.dir.Path, file.Name, len(chunks), len(file.Chunks)) request := &filer_pb.GetFileContentRequest{ - FileId: file.Chunks[0].FileId, + FileId: chunks[0].FileId, } - glog.V(1).Infof("read file content: %v", request) + glog.V(1).Infof("read file content %d chunk %s [%d,%d): %v", len(chunks), + chunks[0].FileId, chunks[0].Offset, chunks[0].Offset+int64(chunks[0].Size), request) resp, err := client.GetFileContent(ctx, request) if err != nil { return err @@ -124,7 +152,7 @@ func (file *File) Flush(ctx context.Context, req *fuse.FlushRequest) error { err := file.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error { - request := &filer_pb.AppendFileChunksRequest{ + request := &filer_pb.SetFileChunksRequest{ Directory: file.dir.Path, Entry: &filer_pb.Entry{ Name: file.Name, @@ -133,7 +161,7 @@ func (file *File) Flush(ctx context.Context, req *fuse.FlushRequest) error { } glog.V(1).Infof("append chunks: %v", request) - if _, err := client.AppendFileChunks(ctx, request); err != nil { + if _, err := client.SetFileChunks(ctx, request); err != nil { return fmt.Errorf("create file: %v", err) } @@ -180,7 +208,7 @@ func (file *File) Write(ctx context.Context, req *fuse.WriteRequest, resp *fuse. return fmt.Errorf("upload result: %v", uploadResult.Error) } - glog.V(1).Infof("uploaded %s/%s to: %v", file.dir.Path, file.Name, fileUrl) + resp.Size = int(uploadResult.Size) file.Chunks = append(file.Chunks, &filer_pb.FileChunk{ FileId: fileId, @@ -189,7 +217,7 @@ func (file *File) Write(ctx context.Context, req *fuse.WriteRequest, resp *fuse. Mtime: time.Now().UnixNano(), }) - resp.Size = int(uploadResult.Size) + glog.V(1).Infof("uploaded %s/%s to: %v, [%d,%d)", file.dir.Path, file.Name, fileUrl, req.Offset, req.Offset+int64(resp.Size)) return nil } diff --git a/weed/pb/filer.proto b/weed/pb/filer.proto index 461a76a58..36b633899 100644 --- a/weed/pb/filer.proto +++ b/weed/pb/filer.proto @@ -21,7 +21,7 @@ service SeaweedFiler { rpc CreateEntry (CreateEntryRequest) returns (CreateEntryResponse) { } - rpc AppendFileChunks (AppendFileChunksRequest) returns (AppendFileChunksResponse) { + rpc SetFileChunks (SetFileChunksRequest) returns (SetFileChunksResponse) { } rpc DeleteEntry (DeleteEntryRequest) returns (DeleteEntryResponse) { @@ -121,9 +121,9 @@ message AssignVolumeResponse { int32 count = 4; } -message AppendFileChunksRequest { +message SetFileChunksRequest { string directory = 1; Entry entry = 2; } -message AppendFileChunksResponse { +message SetFileChunksResponse { } diff --git a/weed/pb/filer_pb/filer.pb.go b/weed/pb/filer_pb/filer.pb.go index fc3bb65c6..6bcecb08c 100644 --- a/weed/pb/filer_pb/filer.pb.go +++ b/weed/pb/filer_pb/filer.pb.go @@ -26,8 +26,8 @@ It has these top-level messages: DeleteEntryResponse AssignVolumeRequest AssignVolumeResponse - AppendFileChunksRequest - AppendFileChunksResponse + SetFileChunksRequest + SetFileChunksResponse */ package filer_pb @@ -475,37 +475,37 @@ func (m *AssignVolumeResponse) GetCount() int32 { return 0 } -type AppendFileChunksRequest struct { +type SetFileChunksRequest struct { Directory string `protobuf:"bytes,1,opt,name=directory" json:"directory,omitempty"` Entry *Entry `protobuf:"bytes,2,opt,name=entry" json:"entry,omitempty"` } -func (m *AppendFileChunksRequest) Reset() { *m = AppendFileChunksRequest{} } -func (m *AppendFileChunksRequest) String() string { return proto.CompactTextString(m) } -func (*AppendFileChunksRequest) ProtoMessage() {} -func (*AppendFileChunksRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{17} } +func (m *SetFileChunksRequest) Reset() { *m = SetFileChunksRequest{} } +func (m *SetFileChunksRequest) String() string { return proto.CompactTextString(m) } +func (*SetFileChunksRequest) ProtoMessage() {} +func (*SetFileChunksRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{17} } -func (m *AppendFileChunksRequest) GetDirectory() string { +func (m *SetFileChunksRequest) GetDirectory() string { if m != nil { return m.Directory } return "" } -func (m *AppendFileChunksRequest) GetEntry() *Entry { +func (m *SetFileChunksRequest) GetEntry() *Entry { if m != nil { return m.Entry } return nil } -type AppendFileChunksResponse struct { +type SetFileChunksResponse struct { } -func (m *AppendFileChunksResponse) Reset() { *m = AppendFileChunksResponse{} } -func (m *AppendFileChunksResponse) String() string { return proto.CompactTextString(m) } -func (*AppendFileChunksResponse) ProtoMessage() {} -func (*AppendFileChunksResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{18} } +func (m *SetFileChunksResponse) Reset() { *m = SetFileChunksResponse{} } +func (m *SetFileChunksResponse) String() string { return proto.CompactTextString(m) } +func (*SetFileChunksResponse) ProtoMessage() {} +func (*SetFileChunksResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{18} } func init() { proto.RegisterType((*LookupDirectoryEntryRequest)(nil), "filer_pb.LookupDirectoryEntryRequest") @@ -525,8 +525,8 @@ func init() { proto.RegisterType((*DeleteEntryResponse)(nil), "filer_pb.DeleteEntryResponse") proto.RegisterType((*AssignVolumeRequest)(nil), "filer_pb.AssignVolumeRequest") proto.RegisterType((*AssignVolumeResponse)(nil), "filer_pb.AssignVolumeResponse") - proto.RegisterType((*AppendFileChunksRequest)(nil), "filer_pb.AppendFileChunksRequest") - proto.RegisterType((*AppendFileChunksResponse)(nil), "filer_pb.AppendFileChunksResponse") + proto.RegisterType((*SetFileChunksRequest)(nil), "filer_pb.SetFileChunksRequest") + proto.RegisterType((*SetFileChunksResponse)(nil), "filer_pb.SetFileChunksResponse") } // Reference imports to suppress errors if they are not otherwise used. @@ -545,7 +545,7 @@ type SeaweedFilerClient interface { GetFileAttributes(ctx context.Context, in *GetFileAttributesRequest, opts ...grpc.CallOption) (*GetFileAttributesResponse, error) GetFileContent(ctx context.Context, in *GetFileContentRequest, opts ...grpc.CallOption) (*GetFileContentResponse, error) CreateEntry(ctx context.Context, in *CreateEntryRequest, opts ...grpc.CallOption) (*CreateEntryResponse, error) - AppendFileChunks(ctx context.Context, in *AppendFileChunksRequest, opts ...grpc.CallOption) (*AppendFileChunksResponse, error) + SetFileChunks(ctx context.Context, in *SetFileChunksRequest, opts ...grpc.CallOption) (*SetFileChunksResponse, error) DeleteEntry(ctx context.Context, in *DeleteEntryRequest, opts ...grpc.CallOption) (*DeleteEntryResponse, error) AssignVolume(ctx context.Context, in *AssignVolumeRequest, opts ...grpc.CallOption) (*AssignVolumeResponse, error) } @@ -603,9 +603,9 @@ func (c *seaweedFilerClient) CreateEntry(ctx context.Context, in *CreateEntryReq return out, nil } -func (c *seaweedFilerClient) AppendFileChunks(ctx context.Context, in *AppendFileChunksRequest, opts ...grpc.CallOption) (*AppendFileChunksResponse, error) { - out := new(AppendFileChunksResponse) - err := grpc.Invoke(ctx, "/filer_pb.SeaweedFiler/AppendFileChunks", in, out, c.cc, opts...) +func (c *seaweedFilerClient) SetFileChunks(ctx context.Context, in *SetFileChunksRequest, opts ...grpc.CallOption) (*SetFileChunksResponse, error) { + out := new(SetFileChunksResponse) + err := grpc.Invoke(ctx, "/filer_pb.SeaweedFiler/SetFileChunks", in, out, c.cc, opts...) if err != nil { return nil, err } @@ -638,7 +638,7 @@ type SeaweedFilerServer interface { GetFileAttributes(context.Context, *GetFileAttributesRequest) (*GetFileAttributesResponse, error) GetFileContent(context.Context, *GetFileContentRequest) (*GetFileContentResponse, error) CreateEntry(context.Context, *CreateEntryRequest) (*CreateEntryResponse, error) - AppendFileChunks(context.Context, *AppendFileChunksRequest) (*AppendFileChunksResponse, error) + SetFileChunks(context.Context, *SetFileChunksRequest) (*SetFileChunksResponse, error) DeleteEntry(context.Context, *DeleteEntryRequest) (*DeleteEntryResponse, error) AssignVolume(context.Context, *AssignVolumeRequest) (*AssignVolumeResponse, error) } @@ -737,20 +737,20 @@ func _SeaweedFiler_CreateEntry_Handler(srv interface{}, ctx context.Context, dec return interceptor(ctx, in, info, handler) } -func _SeaweedFiler_AppendFileChunks_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(AppendFileChunksRequest) +func _SeaweedFiler_SetFileChunks_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(SetFileChunksRequest) if err := dec(in); err != nil { return nil, err } if interceptor == nil { - return srv.(SeaweedFilerServer).AppendFileChunks(ctx, in) + return srv.(SeaweedFilerServer).SetFileChunks(ctx, in) } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/filer_pb.SeaweedFiler/AppendFileChunks", + FullMethod: "/filer_pb.SeaweedFiler/SetFileChunks", } handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(SeaweedFilerServer).AppendFileChunks(ctx, req.(*AppendFileChunksRequest)) + return srv.(SeaweedFilerServer).SetFileChunks(ctx, req.(*SetFileChunksRequest)) } return interceptor(ctx, in, info, handler) } @@ -816,8 +816,8 @@ var _SeaweedFiler_serviceDesc = grpc.ServiceDesc{ Handler: _SeaweedFiler_CreateEntry_Handler, }, { - MethodName: "AppendFileChunks", - Handler: _SeaweedFiler_AppendFileChunks_Handler, + MethodName: "SetFileChunks", + Handler: _SeaweedFiler_SetFileChunks_Handler, }, { MethodName: "DeleteEntry", @@ -835,53 +835,53 @@ var _SeaweedFiler_serviceDesc = grpc.ServiceDesc{ func init() { proto.RegisterFile("filer.proto", fileDescriptor0) } var fileDescriptor0 = []byte{ - // 767 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xac, 0x56, 0x6d, 0x4f, 0xdb, 0x48, - 0x10, 0xc6, 0x38, 0x09, 0x64, 0x12, 0x38, 0x6e, 0x13, 0xc0, 0x17, 0x5e, 0x2e, 0xec, 0x89, 0x13, - 0xa7, 0x93, 0xd0, 0x29, 0xf7, 0xa5, 0x1f, 0x8b, 0x80, 0x56, 0x95, 0xa8, 0x90, 0x8c, 0xa8, 0x54, - 0x55, 0x22, 0x4a, 0xec, 0x49, 0xba, 0xc2, 0xb1, 0x5d, 0x7b, 0xdd, 0x8a, 0x7e, 0xee, 0x5f, 0xe9, - 0x5f, 0xe9, 0xef, 0xaa, 0x76, 0xbd, 0xb1, 0xd7, 0xd8, 0x49, 0x8b, 0xd4, 0x6f, 0xde, 0x79, 0x79, - 0xe6, 0xd9, 0x99, 0x79, 0x36, 0x81, 0xd6, 0x84, 0x79, 0x18, 0x9d, 0x86, 0x51, 0xc0, 0x03, 0xb2, - 0x2e, 0x0f, 0xc3, 0x70, 0x4c, 0xaf, 0x61, 0xef, 0x2a, 0x08, 0xee, 0x93, 0xf0, 0x82, 0x45, 0xe8, - 0xf0, 0x20, 0x7a, 0xb8, 0xf4, 0x79, 0xf4, 0x60, 0xe3, 0x87, 0x04, 0x63, 0x4e, 0xf6, 0xa1, 0xe9, - 0xce, 0x1d, 0x96, 0xd1, 0x37, 0x4e, 0x9a, 0x76, 0x6e, 0x20, 0x04, 0x6a, 0xfe, 0x68, 0x86, 0xd6, - 0xaa, 0x74, 0xc8, 0x6f, 0x7a, 0x09, 0xfb, 0xd5, 0x80, 0x71, 0x18, 0xf8, 0x31, 0x92, 0x63, 0xa8, - 0xa3, 0x30, 0x48, 0xb4, 0xd6, 0xe0, 0xb7, 0xd3, 0x39, 0x95, 0xd3, 0x34, 0x2e, 0xf5, 0xd2, 0x01, - 0x90, 0x2b, 0x16, 0x73, 0x61, 0x63, 0x18, 0xff, 0x14, 0x1d, 0xfa, 0x1c, 0x3a, 0x85, 0x1c, 0x55, - 0xf1, 0x1f, 0x58, 0xc3, 0xd4, 0x64, 0x19, 0x7d, 0xb3, 0xaa, 0xe6, 0xdc, 0x4f, 0xbf, 0x1a, 0x50, - 0x97, 0xa6, 0xec, 0x6a, 0x46, 0x7e, 0x35, 0x72, 0x04, 0x6d, 0x16, 0x0f, 0x73, 0x02, 0xe2, 0xda, - 0xeb, 0x76, 0x8b, 0xc5, 0xd9, 0x55, 0xc9, 0xbf, 0xd0, 0x70, 0xde, 0x27, 0xfe, 0x7d, 0x6c, 0x99, - 0xb2, 0x54, 0x27, 0x2f, 0xf5, 0x82, 0x79, 0x78, 0x2e, 0x7c, 0xb6, 0x0a, 0x21, 0xcf, 0x00, 0x46, - 0x9c, 0x47, 0x6c, 0x9c, 0x70, 0x8c, 0xad, 0x9a, 0xec, 0x87, 0xa5, 0x25, 0x24, 0x31, 0x9e, 0x65, - 0x7e, 0x5b, 0x8b, 0xa5, 0x13, 0x68, 0x66, 0x70, 0x64, 0x17, 0xd6, 0x44, 0xce, 0x90, 0xb9, 0x8a, - 0x6d, 0x43, 0x1c, 0x5f, 0xb9, 0x64, 0x07, 0x1a, 0xc1, 0x64, 0x12, 0x23, 0x97, 0x4c, 0x4d, 0x5b, - 0x9d, 0xc4, 0xdd, 0x62, 0xf6, 0x19, 0x2d, 0xb3, 0x6f, 0x9c, 0xd4, 0x6c, 0xf9, 0x4d, 0xba, 0x50, - 0x9f, 0x71, 0x36, 0x43, 0x49, 0xc3, 0xb4, 0xd3, 0x03, 0xfd, 0x62, 0xc0, 0x66, 0x91, 0x06, 0xd9, - 0x83, 0xa6, 0xac, 0x26, 0x11, 0x0c, 0x89, 0x20, 0xb7, 0xe9, 0xa6, 0x80, 0xb2, 0xaa, 0xa1, 0x64, - 0x29, 0xb3, 0xc0, 0x4d, 0x8b, 0x6e, 0xa4, 0x29, 0xaf, 0x03, 0x17, 0xc9, 0x16, 0x98, 0x09, 0x73, - 0x65, 0xd9, 0x0d, 0x5b, 0x7c, 0x0a, 0xcb, 0x94, 0xb9, 0x56, 0x3d, 0xb5, 0x4c, 0x99, 0x4b, 0x27, - 0x60, 0xbd, 0x44, 0x2e, 0x6e, 0xac, 0xf5, 0x43, 0xad, 0x44, 0xd5, 0xa0, 0x0e, 0x00, 0xc2, 0x51, - 0x84, 0x3e, 0x17, 0xc3, 0x52, 0xdb, 0xd9, 0x4c, 0x2d, 0x17, 0x2c, 0xd2, 0x1b, 0x66, 0xea, 0x0d, - 0xa3, 0xb7, 0xf0, 0x47, 0x45, 0x1d, 0xb5, 0x46, 0xc5, 0x69, 0x19, 0x4f, 0x98, 0xd6, 0x7f, 0xb0, - 0xad, 0x60, 0xcf, 0x03, 0x9f, 0xa3, 0xcf, 0xe7, 0xdc, 0x17, 0x4d, 0x8e, 0x0e, 0x60, 0xe7, 0x71, - 0x86, 0x62, 0x61, 0xc1, 0x9a, 0x93, 0x9a, 0x64, 0x4a, 0xdb, 0x9e, 0x1f, 0xe9, 0x5b, 0x20, 0xe7, - 0x11, 0x8e, 0x38, 0x3e, 0x41, 0xc0, 0x99, 0x18, 0x57, 0x97, 0x8a, 0x71, 0x1b, 0x3a, 0x05, 0xe8, - 0x94, 0x0b, 0x65, 0x40, 0x2e, 0xd0, 0xc3, 0x27, 0x55, 0xac, 0x78, 0x32, 0x4a, 0xba, 0x32, 0x4b, - 0xba, 0x12, 0x0c, 0x0a, 0xa5, 0x14, 0x83, 0x19, 0x74, 0xce, 0xe2, 0x98, 0x4d, 0xfd, 0x37, 0x81, - 0x97, 0xcc, 0x70, 0x4e, 0xa1, 0x0b, 0x75, 0x27, 0x48, 0x54, 0x8b, 0xea, 0x76, 0x7a, 0x20, 0x87, - 0x00, 0x4e, 0xe0, 0x79, 0xe8, 0x70, 0x16, 0xf8, 0x8a, 0x80, 0x66, 0x21, 0x7d, 0x68, 0x45, 0x18, - 0x7a, 0xcc, 0x19, 0xc9, 0x80, 0x74, 0x35, 0x74, 0x13, 0xfd, 0x08, 0xdd, 0x62, 0x39, 0x35, 0x94, - 0x85, 0x0a, 0x14, 0xcb, 0x1d, 0x79, 0xaa, 0x96, 0xf8, 0x94, 0xab, 0x99, 0x8c, 0x3d, 0xe6, 0x0c, - 0x85, 0xc3, 0x54, 0xab, 0x29, 0x2d, 0xb7, 0x91, 0x97, 0x33, 0xaf, 0x69, 0xcc, 0xe9, 0x1d, 0xec, - 0x9e, 0x85, 0x21, 0xfa, 0x6e, 0x26, 0xfa, 0xf8, 0x97, 0xce, 0xb7, 0x07, 0x56, 0x19, 0x3f, 0xbd, - 0xdb, 0xe0, 0x5b, 0x1d, 0xda, 0x37, 0x38, 0xfa, 0x84, 0x28, 0xbd, 0x11, 0x99, 0x42, 0xb7, 0xea, - 0x81, 0x27, 0xc7, 0x39, 0xf8, 0x92, 0x5f, 0x94, 0xde, 0xdf, 0x3f, 0x0a, 0x53, 0xa3, 0x5d, 0x21, - 0x57, 0xd0, 0xd2, 0x9e, 0x73, 0xb2, 0xaf, 0x25, 0x96, 0x7e, 0x19, 0x7a, 0x07, 0x0b, 0xbc, 0x19, - 0xda, 0x1d, 0xfc, 0x5e, 0xd2, 0x36, 0xa1, 0x79, 0xd6, 0xa2, 0x07, 0xa6, 0xf7, 0xd7, 0xd2, 0x98, - 0x0c, 0xff, 0x16, 0x36, 0x8b, 0x92, 0x25, 0x7f, 0x96, 0x12, 0x8b, 0xf2, 0xef, 0xf5, 0x17, 0x07, - 0xe8, 0x4d, 0xd0, 0xa4, 0xa7, 0x37, 0xa1, 0x2c, 0x76, 0xbd, 0x09, 0x55, 0x7a, 0x5d, 0x21, 0xef, - 0x60, 0xeb, 0xf1, 0xa0, 0xc9, 0x51, 0x9e, 0xb4, 0x60, 0xc9, 0x7a, 0x74, 0x59, 0x88, 0x4e, 0x55, - 0xd3, 0xa8, 0x4e, 0xb5, 0xfc, 0x4a, 0xe8, 0x54, 0xab, 0x84, 0xbd, 0x42, 0xae, 0xa1, 0xad, 0x6b, - 0x8d, 0x68, 0x09, 0x15, 0x92, 0xef, 0x1d, 0x2e, 0x72, 0xcf, 0x01, 0xc7, 0x0d, 0xf9, 0xd7, 0xe7, - 0xff, 0xef, 0x01, 0x00, 0x00, 0xff, 0xff, 0x01, 0x52, 0xae, 0x82, 0x09, 0x09, 0x00, 0x00, + // 763 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xac, 0x56, 0xdd, 0x6e, 0xd3, 0x4c, + 0x10, 0xad, 0xeb, 0x24, 0x6d, 0x26, 0x69, 0xbf, 0x8f, 0x4d, 0xda, 0x9a, 0xf4, 0x2f, 0x2c, 0x2a, + 0x2a, 0x42, 0xaa, 0x50, 0xb8, 0xe1, 0x92, 0xaa, 0x2d, 0x08, 0xa9, 0xa8, 0x92, 0xab, 0x22, 0x21, + 0x24, 0xa2, 0xc4, 0x9e, 0x84, 0x55, 0x1d, 0x3b, 0x78, 0xd7, 0xa0, 0x72, 0xcd, 0xab, 0xf0, 0x18, + 0xbc, 0x1b, 0xda, 0xf5, 0xc6, 0x5e, 0x37, 0x4e, 0xa1, 0x12, 0x77, 0xde, 0x99, 0x9d, 0x33, 0x67, + 0x7e, 0xce, 0x26, 0xd0, 0x18, 0xb1, 0x00, 0xe3, 0xa3, 0x69, 0x1c, 0x89, 0x88, 0xac, 0xaa, 0x43, + 0x7f, 0x3a, 0xa4, 0x17, 0xb0, 0x7d, 0x1e, 0x45, 0xd7, 0xc9, 0xf4, 0x94, 0xc5, 0xe8, 0x89, 0x28, + 0xbe, 0x39, 0x0b, 0x45, 0x7c, 0xe3, 0xe2, 0x97, 0x04, 0xb9, 0x20, 0x3b, 0x50, 0xf7, 0x67, 0x0e, + 0xc7, 0xea, 0x5a, 0x87, 0x75, 0x37, 0x37, 0x10, 0x02, 0x95, 0x70, 0x30, 0x41, 0x67, 0x59, 0x39, + 0xd4, 0x37, 0x3d, 0x83, 0x9d, 0x72, 0x40, 0x3e, 0x8d, 0x42, 0x8e, 0xe4, 0x00, 0xaa, 0x28, 0x0d, + 0x0a, 0xad, 0xd1, 0xfb, 0xef, 0x68, 0x46, 0xe5, 0x28, 0xbd, 0x97, 0x7a, 0x69, 0x0f, 0xc8, 0x39, + 0xe3, 0x42, 0xda, 0x18, 0xf2, 0xbf, 0xa2, 0x43, 0x5f, 0x41, 0xab, 0x10, 0xa3, 0x33, 0x3e, 0x85, + 0x15, 0x4c, 0x4d, 0x8e, 0xd5, 0xb5, 0xcb, 0x72, 0xce, 0xfc, 0xf4, 0xa7, 0x05, 0x55, 0x65, 0xca, + 0x4a, 0xb3, 0xf2, 0xd2, 0xc8, 0x23, 0x68, 0x32, 0xde, 0xcf, 0x09, 0xc8, 0xb2, 0x57, 0xdd, 0x06, + 0xe3, 0x59, 0xa9, 0xe4, 0x19, 0xd4, 0xbc, 0xcf, 0x49, 0x78, 0xcd, 0x1d, 0x5b, 0xa5, 0x6a, 0xe5, + 0xa9, 0x5e, 0xb3, 0x00, 0x4f, 0xa4, 0xcf, 0xd5, 0x57, 0xc8, 0x4b, 0x80, 0x81, 0x10, 0x31, 0x1b, + 0x26, 0x02, 0xb9, 0x53, 0x51, 0xfd, 0x70, 0x8c, 0x80, 0x84, 0xe3, 0x71, 0xe6, 0x77, 0x8d, 0xbb, + 0x74, 0x04, 0xf5, 0x0c, 0x8e, 0x6c, 0xc1, 0x8a, 0x8c, 0xe9, 0x33, 0x5f, 0xb3, 0xad, 0xc9, 0xe3, + 0x5b, 0x9f, 0x6c, 0x42, 0x2d, 0x1a, 0x8d, 0x38, 0x0a, 0xc5, 0xd4, 0x76, 0xf5, 0x49, 0xd6, 0xc6, + 0xd9, 0x77, 0x74, 0xec, 0xae, 0x75, 0x58, 0x71, 0xd5, 0x37, 0x69, 0x43, 0x75, 0x22, 0xd8, 0x04, + 0x15, 0x0d, 0xdb, 0x4d, 0x0f, 0xf4, 0x87, 0x05, 0xeb, 0x45, 0x1a, 0x64, 0x1b, 0xea, 0x2a, 0x9b, + 0x42, 0xb0, 0x14, 0x82, 0xda, 0xa6, 0xcb, 0x02, 0xca, 0xb2, 0x81, 0x92, 0x85, 0x4c, 0x22, 0x3f, + 0x4d, 0xba, 0x96, 0x86, 0xbc, 0x8b, 0x7c, 0x24, 0xff, 0x83, 0x9d, 0x30, 0x5f, 0xa5, 0x5d, 0x73, + 0xe5, 0xa7, 0xb4, 0x8c, 0x99, 0xef, 0x54, 0x53, 0xcb, 0x98, 0xf9, 0x74, 0x04, 0xce, 0x1b, 0x14, + 0xb2, 0x62, 0xa3, 0x1f, 0x7a, 0x25, 0xca, 0x06, 0xb5, 0x0b, 0x30, 0x1d, 0xc4, 0x18, 0x0a, 0x39, + 0x2c, 0xbd, 0x9d, 0xf5, 0xd4, 0x72, 0xca, 0x62, 0xb3, 0x61, 0xb6, 0xd9, 0x30, 0x7a, 0x05, 0x0f, + 0x4b, 0xf2, 0xe8, 0x35, 0x2a, 0x4e, 0xcb, 0xba, 0xc7, 0xb4, 0x9e, 0xc3, 0x86, 0x86, 0x3d, 0x89, + 0x42, 0x81, 0xa1, 0x98, 0x71, 0x5f, 0x34, 0x39, 0xda, 0x83, 0xcd, 0xdb, 0x11, 0x9a, 0x85, 0x03, + 0x2b, 0x5e, 0x6a, 0x52, 0x21, 0x4d, 0x77, 0x76, 0xa4, 0x1f, 0x80, 0x9c, 0xc4, 0x38, 0x10, 0x78, + 0x0f, 0x01, 0x67, 0x62, 0x5c, 0xbe, 0x53, 0x8c, 0x1b, 0xd0, 0x2a, 0x40, 0xa7, 0x5c, 0x28, 0x03, + 0x72, 0x8a, 0x01, 0xde, 0x2b, 0x63, 0xc9, 0x93, 0x31, 0xa7, 0x2b, 0x7b, 0x4e, 0x57, 0x92, 0x41, + 0x21, 0x95, 0x66, 0x30, 0x81, 0xd6, 0x31, 0xe7, 0x6c, 0x1c, 0xbe, 0x8f, 0x82, 0x64, 0x82, 0x33, + 0x0a, 0x6d, 0xa8, 0x7a, 0x51, 0xa2, 0x5b, 0x54, 0x75, 0xd3, 0x03, 0xd9, 0x03, 0xf0, 0xa2, 0x20, + 0x40, 0x4f, 0xb0, 0x28, 0xd4, 0x04, 0x0c, 0x0b, 0xe9, 0x42, 0x23, 0xc6, 0x69, 0xc0, 0xbc, 0x81, + 0xba, 0x90, 0xae, 0x86, 0x69, 0xa2, 0x5f, 0xa1, 0x5d, 0x4c, 0xa7, 0x87, 0xb2, 0x50, 0x81, 0x72, + 0xb9, 0xe3, 0x40, 0xe7, 0x92, 0x9f, 0x6a, 0x35, 0x93, 0x61, 0xc0, 0xbc, 0xbe, 0x74, 0xd8, 0x7a, + 0x35, 0x95, 0xe5, 0x2a, 0x0e, 0x72, 0xe6, 0x15, 0x83, 0x39, 0xfd, 0x08, 0xed, 0x4b, 0xbd, 0x0e, + 0xea, 0xe5, 0xf8, 0xa7, 0xc3, 0xdd, 0x82, 0x8d, 0x5b, 0xe0, 0x69, 0x55, 0xbd, 0x5f, 0x55, 0x68, + 0x5e, 0xe2, 0xe0, 0x1b, 0xa2, 0x2f, 0xbd, 0x31, 0x19, 0x43, 0xbb, 0xec, 0x69, 0x27, 0x07, 0x39, + 0xf2, 0x1d, 0xbf, 0x25, 0x9d, 0x27, 0x7f, 0xba, 0xa6, 0x87, 0xba, 0x44, 0xce, 0xa1, 0x61, 0x3c, + 0xe4, 0x64, 0xc7, 0x08, 0x9c, 0xfb, 0x4d, 0xe8, 0xec, 0x2e, 0xf0, 0x66, 0x68, 0x9f, 0xe0, 0xc1, + 0x9c, 0xaa, 0x09, 0xcd, 0xa3, 0x16, 0x3d, 0x2d, 0x9d, 0xc7, 0x77, 0xde, 0xc9, 0xf0, 0xaf, 0x60, + 0xbd, 0x28, 0x56, 0xb2, 0x3f, 0x17, 0x58, 0x14, 0x7e, 0xa7, 0xbb, 0xf8, 0x82, 0xd9, 0x04, 0x43, + 0x74, 0x66, 0x13, 0xe6, 0x65, 0x6e, 0x36, 0xa1, 0x4c, 0xa9, 0x4b, 0xc4, 0x85, 0xb5, 0xc2, 0x94, + 0xc9, 0x5e, 0x1e, 0x51, 0xb6, 0x5b, 0x9d, 0xfd, 0x85, 0x7e, 0x93, 0xa1, 0x21, 0x4a, 0x93, 0xe1, + 0xfc, 0xb3, 0x60, 0x32, 0x2c, 0x53, 0xf2, 0x12, 0xb9, 0x80, 0xa6, 0x29, 0x2e, 0x62, 0x04, 0x94, + 0x68, 0xbc, 0xb3, 0xb7, 0xc8, 0x3d, 0x03, 0x1c, 0xd6, 0xd4, 0x7f, 0x9d, 0x17, 0xbf, 0x03, 0x00, + 0x00, 0xff, 0xff, 0x2a, 0xc2, 0xff, 0xcb, 0xfa, 0x08, 0x00, 0x00, } diff --git a/weed/server/filer_grpc_server.go b/weed/server/filer_grpc_server.go index 6cc5021d6..a279b7dd7 100644 --- a/weed/server/filer_grpc_server.go +++ b/weed/server/filer_grpc_server.go @@ -116,13 +116,34 @@ func (fs *FilerServer) CreateEntry(ctx context.Context, req *filer_pb.CreateEntr return &filer_pb.CreateEntryResponse{}, err } -func (fs *FilerServer) AppendFileChunks(ctx context.Context, req *filer_pb.AppendFileChunksRequest) (*filer_pb.AppendFileChunksResponse, error) { - err := fs.filer.AppendFileChunk( - filer2.FullPath(filepath.Join(req.Directory, req.Entry.Name)), - req.Entry.Chunks, +func (fs *FilerServer) SetFileChunks(ctx context.Context, req *filer_pb.SetFileChunksRequest) (*filer_pb.SetFileChunksResponse, error) { + + fullpath := filepath.Join(req.Directory, req.Entry.Name) + found, entry, err := fs.filer.FindEntry(filer2.FullPath(fullpath)) + if err != nil { + return &filer_pb.SetFileChunksResponse{}, err + } + if !found { + return &filer_pb.SetFileChunksResponse{}, fmt.Errorf("file not found: %s", fullpath) + } + + chunks := append(entry.Chunks, req.Entry.Chunks...) + + chunks, garbages := filer2.CompactFileChunks(chunks) + + err = fs.filer.SetFileChunks( + filer2.FullPath(fullpath), + chunks, ) - return &filer_pb.AppendFileChunksResponse{}, err + if err == nil { + for _, garbage := range garbages { + glog.V(0).Infof("deleting %s old chunk: %v, [%d, %d)", fullpath, garbage.FileId, garbage.Offset, garbage.Offset+int64(garbage.Size)) + operation.DeleteFile(fs.master, garbage.FileId, fs.jwt(garbage.FileId)) + } + } + + return &filer_pb.SetFileChunksResponse{}, err } func (fs *FilerServer) DeleteEntry(ctx context.Context, req *filer_pb.DeleteEntryRequest) (resp *filer_pb.DeleteEntryResponse, err error) { From 9dd228747cd2f6e7f61168e1f394a320bf542623 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Mon, 21 May 2018 01:25:30 -0700 Subject: [PATCH 15/83] filer copy added uid/gid --- weed/command/filer_copy.go | 5 +- weed/filer2/filer.go | 4 +- weed/filer2/filer_structure.go | 3 +- weed/filesys/dir.go | 49 ++++++- weed/filesys/file.go | 4 +- weed/operation/filer/register.go | 4 +- weed/pb/filer.proto | 6 +- weed/pb/filer_pb/filer.pb.go | 158 ++++++++++----------- weed/server/filer_grpc_server.go | 10 +- weed/server/filer_server_handlers_admin.go | 14 ++ 10 files changed, 163 insertions(+), 94 deletions(-) diff --git a/weed/command/filer_copy.go b/weed/command/filer_copy.go index 2b286d3d8..4c203fc94 100644 --- a/weed/command/filer_copy.go +++ b/weed/command/filer_copy.go @@ -68,7 +68,7 @@ func runCopy(cmd *Command, args []string) bool { return false } filerDestination := args[len(args)-1] - fileOrDirs := args[0 : len(args)-1] + fileOrDirs := args[0: len(args)-1] filerUrl, err := url.Parse(filerDestination) if err != nil { @@ -136,7 +136,8 @@ func doEachCopy(fileOrDir string, host string, path string) bool { path = path + fi.Name() } - if err = filer_operation.RegisterFile(host, path, results[0].Fid, parts[0].FileSize, copy.secret); err != nil { + if err = filer_operation.RegisterFile(host, path, results[0].Fid, parts[0].FileSize, + os.Getuid(), os.Getgid(), copy.secret); err != nil { fmt.Printf("Failed to register file %s on %s: %v\n", fileOrDir, host, err) return false } diff --git a/weed/filer2/filer.go b/weed/filer2/filer.go index fd27760bf..facd447f8 100644 --- a/weed/filer2/filer.go +++ b/weed/filer2/filer.go @@ -71,13 +71,13 @@ func (f *Filer) CreateEntry(entry *Entry) (error) { Attr: Attr{ Mtime: now, Crtime: now, - Mode: os.ModeDir | 0660, + Mode: os.ModeDir | 0770, Uid: entry.Uid, Gid: entry.Gid, }, } - glog.V(2).Infof("create directory: %s", dirPath) + glog.V(2).Infof("create directory: %s %v", dirPath, dirEntry.Mode) mkdirErr := f.store.InsertEntry(dirEntry) if mkdirErr != nil { return fmt.Errorf("mkdir %s: %v", dirPath, mkdirErr) diff --git a/weed/filer2/filer_structure.go b/weed/filer2/filer_structure.go index 51de09718..99ca9ce5a 100644 --- a/weed/filer2/filer_structure.go +++ b/weed/filer2/filer_structure.go @@ -6,12 +6,13 @@ import ( "time" "path/filepath" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" + "strings" ) type FullPath string func NewFullPath(dir, name string) FullPath { - if dir == "/" { + if strings.HasSuffix(dir, "/") { return FullPath(dir + name) } return FullPath(dir + "/" + name) diff --git a/weed/filesys/dir.go b/weed/filesys/dir.go index 50b37f883..90c95d4ce 100644 --- a/weed/filesys/dir.go +++ b/weed/filesys/dir.go @@ -12,6 +12,7 @@ import ( "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "time" + "path/filepath" ) type Dir struct { @@ -25,7 +26,53 @@ var _ = fs.Node(&Dir{}) var _ = fs.HandleReadDirAller(&Dir{}) func (dir *Dir) Attr(context context.Context, attr *fuse.Attr) error { - attr.Mode = os.ModeDir | 0777 + + if dir.Path == "/" { + attr.Valid = time.Second + attr.Mode = os.ModeDir | 0777 + return nil + } + + parent, name := filepath.Split(dir.Path) + + var attributes *filer_pb.FuseAttributes + + err := dir.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error { + + request := &filer_pb.GetEntryAttributesRequest{ + Name: name, + ParentDir: parent, + } + + glog.V(1).Infof("read dir attr: %v", request) + resp, err := client.GetEntryAttributes(context, request) + if err != nil { + glog.V(0).Infof("read dir attr %v: %v", request, err) + return err + } + + attributes = resp.Attributes + + return nil + }) + + if err != nil { + return err + } + + // glog.V(1).Infof("dir %s: %v", dir.Path, attributes) + // glog.V(1).Infof("dir %s permission: %v", dir.Path, os.FileMode(attributes.FileMode)) + + attr.Mode = os.FileMode(attributes.FileMode) | os.ModeDir + if dir.Path == "/" && attributes.FileMode == 0 { + attr.Valid = time.Second + } + + attr.Mtime = time.Unix(attributes.Mtime, 0) + attr.Ctime = time.Unix(attributes.Mtime, 0) + attr.Gid = attributes.Gid + attr.Uid = attributes.Uid + return nil } diff --git a/weed/filesys/file.go b/weed/filesys/file.go index d4a9c8b05..81dcba0d9 100644 --- a/weed/filesys/file.go +++ b/weed/filesys/file.go @@ -44,13 +44,13 @@ func (file *File) Attr(context context.Context, attr *fuse.Attr) error { } else { err := file.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error { - request := &filer_pb.GetFileAttributesRequest{ + request := &filer_pb.GetEntryAttributesRequest{ Name: file.Name, ParentDir: file.dir.Path, } glog.V(1).Infof("read file size: %v", request) - resp, err := client.GetFileAttributes(context, request) + resp, err := client.GetEntryAttributes(context, request) if err != nil { glog.V(0).Infof("read file attributes %v: %v", request, err) return err diff --git a/weed/operation/filer/register.go b/weed/operation/filer/register.go index 94e502165..108d93321 100644 --- a/weed/operation/filer/register.go +++ b/weed/operation/filer/register.go @@ -17,7 +17,7 @@ type SubmitResult struct { Error string `json:"error,omitempty"` } -func RegisterFile(filer string, path string, fileId string, fileSize int64, secret security.Secret) error { +func RegisterFile(filer string, path string, fileId string, fileSize int64, uid, gid int, secret security.Secret) error { // TODO: jwt need to be used _ = security.GenJwt(secret, fileId) @@ -25,6 +25,8 @@ func RegisterFile(filer string, path string, fileId string, fileSize int64, secr values.Add("path", path) values.Add("fileId", fileId) values.Add("fileSize", strconv.FormatInt(fileSize, 10)) + values.Add("uid", strconv.Itoa(uid)) + values.Add("gid", strconv.Itoa(gid)) _, err := util.Post("http://"+filer+"/admin/register", values) if err != nil { return fmt.Errorf("Failed to register path %s on filer %s to file id %s : %v", path, filer, fileId, err) diff --git a/weed/pb/filer.proto b/weed/pb/filer.proto index 36b633899..1f440d19c 100644 --- a/weed/pb/filer.proto +++ b/weed/pb/filer.proto @@ -12,7 +12,7 @@ service SeaweedFiler { rpc ListEntries (ListEntriesRequest) returns (ListEntriesResponse) { } - rpc GetFileAttributes (GetFileAttributesRequest) returns (GetFileAttributesResponse) { + rpc GetEntryAttributes (GetEntryAttributesRequest) returns (GetEntryAttributesResponse) { } rpc GetFileContent (GetFileContentRequest) returns (GetFileContentResponse) { @@ -73,13 +73,13 @@ message FuseAttributes { uint32 gid = 5; } -message GetFileAttributesRequest { +message GetEntryAttributesRequest { string name = 1; string parent_dir = 2; string file_id = 3; } -message GetFileAttributesResponse { +message GetEntryAttributesResponse { FuseAttributes attributes = 1; } diff --git a/weed/pb/filer_pb/filer.pb.go b/weed/pb/filer_pb/filer.pb.go index 6bcecb08c..a8a20ffe6 100644 --- a/weed/pb/filer_pb/filer.pb.go +++ b/weed/pb/filer_pb/filer.pb.go @@ -16,8 +16,8 @@ It has these top-level messages: Entry FileChunk FuseAttributes - GetFileAttributesRequest - GetFileAttributesResponse + GetEntryAttributesRequest + GetEntryAttributesResponse GetFileContentRequest GetFileContentResponse CreateEntryRequest @@ -251,48 +251,48 @@ func (m *FuseAttributes) GetGid() uint32 { return 0 } -type GetFileAttributesRequest struct { +type GetEntryAttributesRequest struct { Name string `protobuf:"bytes,1,opt,name=name" json:"name,omitempty"` ParentDir string `protobuf:"bytes,2,opt,name=parent_dir,json=parentDir" json:"parent_dir,omitempty"` FileId string `protobuf:"bytes,3,opt,name=file_id,json=fileId" json:"file_id,omitempty"` } -func (m *GetFileAttributesRequest) Reset() { *m = GetFileAttributesRequest{} } -func (m *GetFileAttributesRequest) String() string { return proto.CompactTextString(m) } -func (*GetFileAttributesRequest) ProtoMessage() {} -func (*GetFileAttributesRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{7} } +func (m *GetEntryAttributesRequest) Reset() { *m = GetEntryAttributesRequest{} } +func (m *GetEntryAttributesRequest) String() string { return proto.CompactTextString(m) } +func (*GetEntryAttributesRequest) ProtoMessage() {} +func (*GetEntryAttributesRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{7} } -func (m *GetFileAttributesRequest) GetName() string { +func (m *GetEntryAttributesRequest) GetName() string { if m != nil { return m.Name } return "" } -func (m *GetFileAttributesRequest) GetParentDir() string { +func (m *GetEntryAttributesRequest) GetParentDir() string { if m != nil { return m.ParentDir } return "" } -func (m *GetFileAttributesRequest) GetFileId() string { +func (m *GetEntryAttributesRequest) GetFileId() string { if m != nil { return m.FileId } return "" } -type GetFileAttributesResponse struct { +type GetEntryAttributesResponse struct { Attributes *FuseAttributes `protobuf:"bytes,1,opt,name=attributes" json:"attributes,omitempty"` } -func (m *GetFileAttributesResponse) Reset() { *m = GetFileAttributesResponse{} } -func (m *GetFileAttributesResponse) String() string { return proto.CompactTextString(m) } -func (*GetFileAttributesResponse) ProtoMessage() {} -func (*GetFileAttributesResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{8} } +func (m *GetEntryAttributesResponse) Reset() { *m = GetEntryAttributesResponse{} } +func (m *GetEntryAttributesResponse) String() string { return proto.CompactTextString(m) } +func (*GetEntryAttributesResponse) ProtoMessage() {} +func (*GetEntryAttributesResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{8} } -func (m *GetFileAttributesResponse) GetAttributes() *FuseAttributes { +func (m *GetEntryAttributesResponse) GetAttributes() *FuseAttributes { if m != nil { return m.Attributes } @@ -515,8 +515,8 @@ func init() { proto.RegisterType((*Entry)(nil), "filer_pb.Entry") proto.RegisterType((*FileChunk)(nil), "filer_pb.FileChunk") proto.RegisterType((*FuseAttributes)(nil), "filer_pb.FuseAttributes") - proto.RegisterType((*GetFileAttributesRequest)(nil), "filer_pb.GetFileAttributesRequest") - proto.RegisterType((*GetFileAttributesResponse)(nil), "filer_pb.GetFileAttributesResponse") + proto.RegisterType((*GetEntryAttributesRequest)(nil), "filer_pb.GetEntryAttributesRequest") + proto.RegisterType((*GetEntryAttributesResponse)(nil), "filer_pb.GetEntryAttributesResponse") proto.RegisterType((*GetFileContentRequest)(nil), "filer_pb.GetFileContentRequest") proto.RegisterType((*GetFileContentResponse)(nil), "filer_pb.GetFileContentResponse") proto.RegisterType((*CreateEntryRequest)(nil), "filer_pb.CreateEntryRequest") @@ -542,7 +542,7 @@ const _ = grpc.SupportPackageIsVersion4 type SeaweedFilerClient interface { LookupDirectoryEntry(ctx context.Context, in *LookupDirectoryEntryRequest, opts ...grpc.CallOption) (*LookupDirectoryEntryResponse, error) ListEntries(ctx context.Context, in *ListEntriesRequest, opts ...grpc.CallOption) (*ListEntriesResponse, error) - GetFileAttributes(ctx context.Context, in *GetFileAttributesRequest, opts ...grpc.CallOption) (*GetFileAttributesResponse, error) + GetEntryAttributes(ctx context.Context, in *GetEntryAttributesRequest, opts ...grpc.CallOption) (*GetEntryAttributesResponse, error) GetFileContent(ctx context.Context, in *GetFileContentRequest, opts ...grpc.CallOption) (*GetFileContentResponse, error) CreateEntry(ctx context.Context, in *CreateEntryRequest, opts ...grpc.CallOption) (*CreateEntryResponse, error) SetFileChunks(ctx context.Context, in *SetFileChunksRequest, opts ...grpc.CallOption) (*SetFileChunksResponse, error) @@ -576,9 +576,9 @@ func (c *seaweedFilerClient) ListEntries(ctx context.Context, in *ListEntriesReq return out, nil } -func (c *seaweedFilerClient) GetFileAttributes(ctx context.Context, in *GetFileAttributesRequest, opts ...grpc.CallOption) (*GetFileAttributesResponse, error) { - out := new(GetFileAttributesResponse) - err := grpc.Invoke(ctx, "/filer_pb.SeaweedFiler/GetFileAttributes", in, out, c.cc, opts...) +func (c *seaweedFilerClient) GetEntryAttributes(ctx context.Context, in *GetEntryAttributesRequest, opts ...grpc.CallOption) (*GetEntryAttributesResponse, error) { + out := new(GetEntryAttributesResponse) + err := grpc.Invoke(ctx, "/filer_pb.SeaweedFiler/GetEntryAttributes", in, out, c.cc, opts...) if err != nil { return nil, err } @@ -635,7 +635,7 @@ func (c *seaweedFilerClient) AssignVolume(ctx context.Context, in *AssignVolumeR type SeaweedFilerServer interface { LookupDirectoryEntry(context.Context, *LookupDirectoryEntryRequest) (*LookupDirectoryEntryResponse, error) ListEntries(context.Context, *ListEntriesRequest) (*ListEntriesResponse, error) - GetFileAttributes(context.Context, *GetFileAttributesRequest) (*GetFileAttributesResponse, error) + GetEntryAttributes(context.Context, *GetEntryAttributesRequest) (*GetEntryAttributesResponse, error) GetFileContent(context.Context, *GetFileContentRequest) (*GetFileContentResponse, error) CreateEntry(context.Context, *CreateEntryRequest) (*CreateEntryResponse, error) SetFileChunks(context.Context, *SetFileChunksRequest) (*SetFileChunksResponse, error) @@ -683,20 +683,20 @@ func _SeaweedFiler_ListEntries_Handler(srv interface{}, ctx context.Context, dec return interceptor(ctx, in, info, handler) } -func _SeaweedFiler_GetFileAttributes_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(GetFileAttributesRequest) +func _SeaweedFiler_GetEntryAttributes_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(GetEntryAttributesRequest) if err := dec(in); err != nil { return nil, err } if interceptor == nil { - return srv.(SeaweedFilerServer).GetFileAttributes(ctx, in) + return srv.(SeaweedFilerServer).GetEntryAttributes(ctx, in) } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/filer_pb.SeaweedFiler/GetFileAttributes", + FullMethod: "/filer_pb.SeaweedFiler/GetEntryAttributes", } handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(SeaweedFilerServer).GetFileAttributes(ctx, req.(*GetFileAttributesRequest)) + return srv.(SeaweedFilerServer).GetEntryAttributes(ctx, req.(*GetEntryAttributesRequest)) } return interceptor(ctx, in, info, handler) } @@ -804,8 +804,8 @@ var _SeaweedFiler_serviceDesc = grpc.ServiceDesc{ Handler: _SeaweedFiler_ListEntries_Handler, }, { - MethodName: "GetFileAttributes", - Handler: _SeaweedFiler_GetFileAttributes_Handler, + MethodName: "GetEntryAttributes", + Handler: _SeaweedFiler_GetEntryAttributes_Handler, }, { MethodName: "GetFileContent", @@ -835,53 +835,53 @@ var _SeaweedFiler_serviceDesc = grpc.ServiceDesc{ func init() { proto.RegisterFile("filer.proto", fileDescriptor0) } var fileDescriptor0 = []byte{ - // 763 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xac, 0x56, 0xdd, 0x6e, 0xd3, 0x4c, - 0x10, 0xad, 0xeb, 0x24, 0x6d, 0x26, 0x69, 0xbf, 0x8f, 0x4d, 0xda, 0x9a, 0xf4, 0x2f, 0x2c, 0x2a, - 0x2a, 0x42, 0xaa, 0x50, 0xb8, 0xe1, 0x92, 0xaa, 0x2d, 0x08, 0xa9, 0xa8, 0x92, 0xab, 0x22, 0x21, - 0x24, 0xa2, 0xc4, 0x9e, 0x84, 0x55, 0x1d, 0x3b, 0x78, 0xd7, 0xa0, 0x72, 0xcd, 0xab, 0xf0, 0x18, - 0xbc, 0x1b, 0xda, 0xf5, 0xc6, 0x5e, 0x37, 0x4e, 0xa1, 0x12, 0x77, 0xde, 0x99, 0x9d, 0x33, 0x67, - 0x7e, 0xce, 0x26, 0xd0, 0x18, 0xb1, 0x00, 0xe3, 0xa3, 0x69, 0x1c, 0x89, 0x88, 0xac, 0xaa, 0x43, - 0x7f, 0x3a, 0xa4, 0x17, 0xb0, 0x7d, 0x1e, 0x45, 0xd7, 0xc9, 0xf4, 0x94, 0xc5, 0xe8, 0x89, 0x28, - 0xbe, 0x39, 0x0b, 0x45, 0x7c, 0xe3, 0xe2, 0x97, 0x04, 0xb9, 0x20, 0x3b, 0x50, 0xf7, 0x67, 0x0e, - 0xc7, 0xea, 0x5a, 0x87, 0x75, 0x37, 0x37, 0x10, 0x02, 0x95, 0x70, 0x30, 0x41, 0x67, 0x59, 0x39, - 0xd4, 0x37, 0x3d, 0x83, 0x9d, 0x72, 0x40, 0x3e, 0x8d, 0x42, 0x8e, 0xe4, 0x00, 0xaa, 0x28, 0x0d, - 0x0a, 0xad, 0xd1, 0xfb, 0xef, 0x68, 0x46, 0xe5, 0x28, 0xbd, 0x97, 0x7a, 0x69, 0x0f, 0xc8, 0x39, - 0xe3, 0x42, 0xda, 0x18, 0xf2, 0xbf, 0xa2, 0x43, 0x5f, 0x41, 0xab, 0x10, 0xa3, 0x33, 0x3e, 0x85, - 0x15, 0x4c, 0x4d, 0x8e, 0xd5, 0xb5, 0xcb, 0x72, 0xce, 0xfc, 0xf4, 0xa7, 0x05, 0x55, 0x65, 0xca, - 0x4a, 0xb3, 0xf2, 0xd2, 0xc8, 0x23, 0x68, 0x32, 0xde, 0xcf, 0x09, 0xc8, 0xb2, 0x57, 0xdd, 0x06, - 0xe3, 0x59, 0xa9, 0xe4, 0x19, 0xd4, 0xbc, 0xcf, 0x49, 0x78, 0xcd, 0x1d, 0x5b, 0xa5, 0x6a, 0xe5, - 0xa9, 0x5e, 0xb3, 0x00, 0x4f, 0xa4, 0xcf, 0xd5, 0x57, 0xc8, 0x4b, 0x80, 0x81, 0x10, 0x31, 0x1b, - 0x26, 0x02, 0xb9, 0x53, 0x51, 0xfd, 0x70, 0x8c, 0x80, 0x84, 0xe3, 0x71, 0xe6, 0x77, 0x8d, 0xbb, - 0x74, 0x04, 0xf5, 0x0c, 0x8e, 0x6c, 0xc1, 0x8a, 0x8c, 0xe9, 0x33, 0x5f, 0xb3, 0xad, 0xc9, 0xe3, - 0x5b, 0x9f, 0x6c, 0x42, 0x2d, 0x1a, 0x8d, 0x38, 0x0a, 0xc5, 0xd4, 0x76, 0xf5, 0x49, 0xd6, 0xc6, - 0xd9, 0x77, 0x74, 0xec, 0xae, 0x75, 0x58, 0x71, 0xd5, 0x37, 0x69, 0x43, 0x75, 0x22, 0xd8, 0x04, - 0x15, 0x0d, 0xdb, 0x4d, 0x0f, 0xf4, 0x87, 0x05, 0xeb, 0x45, 0x1a, 0x64, 0x1b, 0xea, 0x2a, 0x9b, - 0x42, 0xb0, 0x14, 0x82, 0xda, 0xa6, 0xcb, 0x02, 0xca, 0xb2, 0x81, 0x92, 0x85, 0x4c, 0x22, 0x3f, - 0x4d, 0xba, 0x96, 0x86, 0xbc, 0x8b, 0x7c, 0x24, 0xff, 0x83, 0x9d, 0x30, 0x5f, 0xa5, 0x5d, 0x73, - 0xe5, 0xa7, 0xb4, 0x8c, 0x99, 0xef, 0x54, 0x53, 0xcb, 0x98, 0xf9, 0x74, 0x04, 0xce, 0x1b, 0x14, - 0xb2, 0x62, 0xa3, 0x1f, 0x7a, 0x25, 0xca, 0x06, 0xb5, 0x0b, 0x30, 0x1d, 0xc4, 0x18, 0x0a, 0x39, - 0x2c, 0xbd, 0x9d, 0xf5, 0xd4, 0x72, 0xca, 0x62, 0xb3, 0x61, 0xb6, 0xd9, 0x30, 0x7a, 0x05, 0x0f, - 0x4b, 0xf2, 0xe8, 0x35, 0x2a, 0x4e, 0xcb, 0xba, 0xc7, 0xb4, 0x9e, 0xc3, 0x86, 0x86, 0x3d, 0x89, - 0x42, 0x81, 0xa1, 0x98, 0x71, 0x5f, 0x34, 0x39, 0xda, 0x83, 0xcd, 0xdb, 0x11, 0x9a, 0x85, 0x03, - 0x2b, 0x5e, 0x6a, 0x52, 0x21, 0x4d, 0x77, 0x76, 0xa4, 0x1f, 0x80, 0x9c, 0xc4, 0x38, 0x10, 0x78, - 0x0f, 0x01, 0x67, 0x62, 0x5c, 0xbe, 0x53, 0x8c, 0x1b, 0xd0, 0x2a, 0x40, 0xa7, 0x5c, 0x28, 0x03, - 0x72, 0x8a, 0x01, 0xde, 0x2b, 0x63, 0xc9, 0x93, 0x31, 0xa7, 0x2b, 0x7b, 0x4e, 0x57, 0x92, 0x41, - 0x21, 0x95, 0x66, 0x30, 0x81, 0xd6, 0x31, 0xe7, 0x6c, 0x1c, 0xbe, 0x8f, 0x82, 0x64, 0x82, 0x33, - 0x0a, 0x6d, 0xa8, 0x7a, 0x51, 0xa2, 0x5b, 0x54, 0x75, 0xd3, 0x03, 0xd9, 0x03, 0xf0, 0xa2, 0x20, - 0x40, 0x4f, 0xb0, 0x28, 0xd4, 0x04, 0x0c, 0x0b, 0xe9, 0x42, 0x23, 0xc6, 0x69, 0xc0, 0xbc, 0x81, - 0xba, 0x90, 0xae, 0x86, 0x69, 0xa2, 0x5f, 0xa1, 0x5d, 0x4c, 0xa7, 0x87, 0xb2, 0x50, 0x81, 0x72, - 0xb9, 0xe3, 0x40, 0xe7, 0x92, 0x9f, 0x6a, 0x35, 0x93, 0x61, 0xc0, 0xbc, 0xbe, 0x74, 0xd8, 0x7a, - 0x35, 0x95, 0xe5, 0x2a, 0x0e, 0x72, 0xe6, 0x15, 0x83, 0x39, 0xfd, 0x08, 0xed, 0x4b, 0xbd, 0x0e, - 0xea, 0xe5, 0xf8, 0xa7, 0xc3, 0xdd, 0x82, 0x8d, 0x5b, 0xe0, 0x69, 0x55, 0xbd, 0x5f, 0x55, 0x68, - 0x5e, 0xe2, 0xe0, 0x1b, 0xa2, 0x2f, 0xbd, 0x31, 0x19, 0x43, 0xbb, 0xec, 0x69, 0x27, 0x07, 0x39, - 0xf2, 0x1d, 0xbf, 0x25, 0x9d, 0x27, 0x7f, 0xba, 0xa6, 0x87, 0xba, 0x44, 0xce, 0xa1, 0x61, 0x3c, - 0xe4, 0x64, 0xc7, 0x08, 0x9c, 0xfb, 0x4d, 0xe8, 0xec, 0x2e, 0xf0, 0x66, 0x68, 0x9f, 0xe0, 0xc1, - 0x9c, 0xaa, 0x09, 0xcd, 0xa3, 0x16, 0x3d, 0x2d, 0x9d, 0xc7, 0x77, 0xde, 0xc9, 0xf0, 0xaf, 0x60, - 0xbd, 0x28, 0x56, 0xb2, 0x3f, 0x17, 0x58, 0x14, 0x7e, 0xa7, 0xbb, 0xf8, 0x82, 0xd9, 0x04, 0x43, - 0x74, 0x66, 0x13, 0xe6, 0x65, 0x6e, 0x36, 0xa1, 0x4c, 0xa9, 0x4b, 0xc4, 0x85, 0xb5, 0xc2, 0x94, - 0xc9, 0x5e, 0x1e, 0x51, 0xb6, 0x5b, 0x9d, 0xfd, 0x85, 0x7e, 0x93, 0xa1, 0x21, 0x4a, 0x93, 0xe1, - 0xfc, 0xb3, 0x60, 0x32, 0x2c, 0x53, 0xf2, 0x12, 0xb9, 0x80, 0xa6, 0x29, 0x2e, 0x62, 0x04, 0x94, - 0x68, 0xbc, 0xb3, 0xb7, 0xc8, 0x3d, 0x03, 0x1c, 0xd6, 0xd4, 0x7f, 0x9d, 0x17, 0xbf, 0x03, 0x00, - 0x00, 0xff, 0xff, 0x2a, 0xc2, 0xff, 0xcb, 0xfa, 0x08, 0x00, 0x00, + // 765 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xac, 0x56, 0xdd, 0x4e, 0xdb, 0x4a, + 0x10, 0xc6, 0x38, 0x09, 0x64, 0x12, 0x38, 0x47, 0x93, 0x00, 0x3e, 0xe1, 0x2f, 0xc7, 0x2d, 0x15, + 0x55, 0x25, 0x54, 0xa5, 0x37, 0xbd, 0x2c, 0x02, 0x8a, 0x2a, 0x51, 0x21, 0x19, 0x81, 0x54, 0xf5, + 0x22, 0x4a, 0xec, 0x49, 0xba, 0xc2, 0xb1, 0x53, 0x7b, 0xdd, 0x8a, 0x5e, 0xf7, 0x55, 0xfa, 0x1e, + 0x7d, 0xb4, 0x6a, 0xd7, 0x1b, 0x7b, 0x4d, 0x1c, 0x5a, 0xa4, 0xde, 0x79, 0xe7, 0xef, 0xfb, 0x76, + 0x66, 0xbe, 0x4d, 0xa0, 0x31, 0x62, 0x3e, 0x45, 0x47, 0xd3, 0x28, 0xe4, 0x21, 0xae, 0xca, 0x43, + 0x7f, 0x3a, 0xb4, 0x2f, 0x61, 0xfb, 0x22, 0x0c, 0x6f, 0x93, 0xe9, 0x29, 0x8b, 0xc8, 0xe5, 0x61, + 0x74, 0x77, 0x16, 0xf0, 0xe8, 0xce, 0xa1, 0xcf, 0x09, 0xc5, 0x1c, 0x77, 0xa0, 0xee, 0xcd, 0x1c, + 0x96, 0xd1, 0x35, 0x0e, 0xeb, 0x4e, 0x6e, 0x40, 0x84, 0x4a, 0x30, 0x98, 0x90, 0xb5, 0x2c, 0x1d, + 0xf2, 0xdb, 0x3e, 0x83, 0x9d, 0xf2, 0x82, 0xf1, 0x34, 0x0c, 0x62, 0xc2, 0x03, 0xa8, 0x92, 0x30, + 0xc8, 0x6a, 0x8d, 0xde, 0x3f, 0x47, 0x33, 0x2a, 0x47, 0x69, 0x5c, 0xea, 0xb5, 0x7b, 0x80, 0x17, + 0x2c, 0xe6, 0xc2, 0xc6, 0x28, 0xfe, 0x23, 0x3a, 0xf6, 0x1b, 0x68, 0x15, 0x72, 0x14, 0xe2, 0x73, + 0x58, 0xa1, 0xd4, 0x64, 0x19, 0x5d, 0xb3, 0x0c, 0x73, 0xe6, 0xb7, 0x7f, 0x18, 0x50, 0x95, 0xa6, + 0xec, 0x6a, 0x46, 0x7e, 0x35, 0xfc, 0x1f, 0x9a, 0x2c, 0xee, 0xe7, 0x04, 0xc4, 0xb5, 0x57, 0x9d, + 0x06, 0x8b, 0xb3, 0xab, 0xe2, 0x0b, 0xa8, 0xb9, 0x9f, 0x92, 0xe0, 0x36, 0xb6, 0x4c, 0x09, 0xd5, + 0xca, 0xa1, 0xde, 0x32, 0x9f, 0x4e, 0x84, 0xcf, 0x51, 0x21, 0xf8, 0x1a, 0x60, 0xc0, 0x79, 0xc4, + 0x86, 0x09, 0xa7, 0xd8, 0xaa, 0xc8, 0x7e, 0x58, 0x5a, 0x42, 0x12, 0xd3, 0x71, 0xe6, 0x77, 0xb4, + 0x58, 0x7b, 0x04, 0xf5, 0xac, 0x1c, 0x6e, 0xc1, 0x8a, 0xc8, 0xe9, 0x33, 0x4f, 0xb1, 0xad, 0x89, + 0xe3, 0x3b, 0x0f, 0x37, 0xa1, 0x16, 0x8e, 0x46, 0x31, 0x71, 0xc9, 0xd4, 0x74, 0xd4, 0x49, 0xdc, + 0x2d, 0x66, 0xdf, 0xc8, 0x32, 0xbb, 0xc6, 0x61, 0xc5, 0x91, 0xdf, 0xd8, 0x86, 0xea, 0x84, 0xb3, + 0x09, 0x49, 0x1a, 0xa6, 0x93, 0x1e, 0xec, 0xef, 0x06, 0xac, 0x17, 0x69, 0xe0, 0x36, 0xd4, 0x25, + 0x9a, 0xac, 0x60, 0xc8, 0x0a, 0x72, 0x9b, 0xae, 0x0a, 0x55, 0x96, 0xb5, 0x2a, 0x59, 0xca, 0x24, + 0xf4, 0x52, 0xd0, 0xb5, 0x34, 0xe5, 0x7d, 0xe8, 0x11, 0xfe, 0x0b, 0x66, 0xc2, 0x3c, 0x09, 0xbb, + 0xe6, 0x88, 0x4f, 0x61, 0x19, 0x33, 0xcf, 0xaa, 0xa6, 0x96, 0x31, 0xf3, 0xec, 0x31, 0xfc, 0x77, + 0x4e, 0x72, 0xae, 0x77, 0x5a, 0x43, 0xd4, 0x4e, 0x94, 0x4d, 0x6a, 0x17, 0x60, 0x3a, 0x88, 0x28, + 0xe0, 0x62, 0x5a, 0x6a, 0x3d, 0xeb, 0xa9, 0xe5, 0x94, 0x45, 0x7a, 0xc7, 0x4c, 0xbd, 0x63, 0xf6, + 0x0d, 0x74, 0xca, 0x80, 0xd4, 0x22, 0x15, 0xe7, 0x65, 0x3c, 0x62, 0x5e, 0x2f, 0x61, 0xe3, 0x9c, + 0xb8, 0x1c, 0x59, 0x18, 0x70, 0x0a, 0xf8, 0x8c, 0xfc, 0xa2, 0xd9, 0xd9, 0x3d, 0xd8, 0xbc, 0x9f, + 0xa1, 0x58, 0x58, 0xb0, 0xe2, 0xa6, 0x26, 0x99, 0xd2, 0x74, 0x66, 0x47, 0xfb, 0x03, 0xe0, 0x49, + 0x44, 0x03, 0x4e, 0x8f, 0x90, 0x70, 0x26, 0xc7, 0xe5, 0x07, 0xe5, 0xb8, 0x01, 0xad, 0x42, 0xe9, + 0x94, 0x8b, 0xcd, 0x00, 0x4f, 0xc9, 0xa7, 0x47, 0x21, 0x96, 0x3c, 0x1a, 0x73, 0xca, 0x32, 0xe7, + 0x94, 0x25, 0x18, 0x14, 0xa0, 0x14, 0x83, 0x09, 0xb4, 0x8e, 0xe3, 0x98, 0x8d, 0x83, 0x9b, 0xd0, + 0x4f, 0x26, 0x34, 0xa3, 0xd0, 0x86, 0xaa, 0x1b, 0x26, 0xaa, 0x45, 0x55, 0x27, 0x3d, 0xe0, 0x1e, + 0x80, 0x1b, 0xfa, 0x3e, 0xb9, 0x9c, 0x85, 0x81, 0x22, 0xa0, 0x59, 0xb0, 0x0b, 0x8d, 0x88, 0xa6, + 0x3e, 0x73, 0x07, 0x32, 0x20, 0xdd, 0x0d, 0xdd, 0x64, 0x7f, 0x81, 0x76, 0x11, 0x4e, 0x0d, 0x65, + 0xa1, 0x06, 0xc5, 0x7a, 0x47, 0xbe, 0xc2, 0x12, 0x9f, 0x72, 0x37, 0x93, 0xa1, 0xcf, 0xdc, 0xbe, + 0x70, 0x98, 0x6a, 0x37, 0xa5, 0xe5, 0x3a, 0xf2, 0x73, 0xe6, 0x15, 0x8d, 0xb9, 0xfd, 0x11, 0xda, + 0x57, 0x6a, 0x1d, 0xe4, 0xdb, 0xf1, 0x57, 0x87, 0xbb, 0x05, 0x1b, 0xf7, 0x8a, 0xa7, 0xb7, 0xea, + 0xfd, 0xac, 0x42, 0xf3, 0x8a, 0x06, 0x5f, 0x89, 0x3c, 0xe1, 0x8d, 0x70, 0x0c, 0xed, 0xb2, 0xc7, + 0x1d, 0x0f, 0xf2, 0xca, 0x0f, 0xfc, 0x9a, 0x74, 0x9e, 0xfd, 0x2e, 0x4c, 0x0d, 0x75, 0x09, 0x2f, + 0xa0, 0xa1, 0x3d, 0xe5, 0xb8, 0xa3, 0x25, 0xce, 0xfd, 0x2a, 0x74, 0x76, 0x17, 0x78, 0xb3, 0x6a, + 0x03, 0xc0, 0x79, 0x59, 0xe3, 0x93, 0x3c, 0x6d, 0xe1, 0xeb, 0xd2, 0x79, 0xfa, 0x70, 0x50, 0x06, + 0x71, 0x0d, 0xeb, 0x45, 0xbd, 0xe2, 0x7e, 0x21, 0x73, 0x5e, 0xfb, 0x9d, 0xee, 0xe2, 0x00, 0xbd, + 0x0f, 0x9a, 0xee, 0xf4, 0x3e, 0xcc, 0x2b, 0x5d, 0xef, 0x43, 0x99, 0x58, 0x97, 0xd0, 0x81, 0xb5, + 0xc2, 0xa0, 0x71, 0x2f, 0xcf, 0x28, 0x5b, 0xaf, 0xce, 0xfe, 0x42, 0xbf, 0xce, 0x50, 0xd3, 0xa5, + 0xce, 0x70, 0xfe, 0x65, 0xd0, 0x19, 0x96, 0x89, 0x79, 0x09, 0x2f, 0xa1, 0xa9, 0xeb, 0x0b, 0xb5, + 0x84, 0x12, 0x99, 0x77, 0xf6, 0x16, 0xb9, 0x67, 0x05, 0x87, 0x35, 0xf9, 0x87, 0xe7, 0xd5, 0xaf, + 0x00, 0x00, 0x00, 0xff, 0xff, 0xc7, 0x51, 0xe7, 0x2b, 0xff, 0x08, 0x00, 0x00, } diff --git a/weed/server/filer_grpc_server.go b/weed/server/filer_grpc_server.go index a279b7dd7..3f05d777b 100644 --- a/weed/server/filer_grpc_server.go +++ b/weed/server/filer_grpc_server.go @@ -59,11 +59,13 @@ func (fs *FilerServer) ListEntries(ctx context.Context, req *filer_pb.ListEntrie return resp, nil } -func (fs *FilerServer) GetFileAttributes(ctx context.Context, req *filer_pb.GetFileAttributesRequest) (*filer_pb.GetFileAttributesResponse, error) { +func (fs *FilerServer) GetEntryAttributes(ctx context.Context, req *filer_pb.GetEntryAttributesRequest) (*filer_pb.GetEntryAttributesResponse, error) { attributes := &filer_pb.FuseAttributes{} - found, entry, err := fs.filer.FindEntry(filer2.NewFullPath(req.ParentDir, req.Name)) + fullpath := filer2.NewFullPath(req.ParentDir, req.Name) + + found, entry, err := fs.filer.FindEntry(fullpath) if err != nil { return nil, err } @@ -77,7 +79,9 @@ func (fs *FilerServer) GetFileAttributes(ctx context.Context, req *filer_pb.GetF attributes.Mtime = entry.Mtime.Unix() } - return &filer_pb.GetFileAttributesResponse{ + glog.V(0).Infof("GetEntryAttributes %v: %+v", fullpath, attributes) + + return &filer_pb.GetEntryAttributesResponse{ Attributes: attributes, }, nil } diff --git a/weed/server/filer_server_handlers_admin.go b/weed/server/filer_server_handlers_admin.go index 06d041398..7f17f936d 100644 --- a/weed/server/filer_server_handlers_admin.go +++ b/weed/server/filer_server_handlers_admin.go @@ -19,10 +19,24 @@ func (fs *FilerServer) registerHandler(w http.ResponseWriter, r *http.Request) { writeJsonError(w, r, http.StatusInternalServerError, err) return } + uid, err := strconv.ParseUint(r.FormValue("uid"), 10, 64) + if err != nil && r.FormValue("uid") != "" { + glog.V(0).Infof("register %s to %s parse uid %s: %v", fileId, path, r.FormValue("uid"), err) + writeJsonError(w, r, http.StatusInternalServerError, err) + return + } + gid, err := strconv.ParseUint(r.FormValue("gid"), 10, 64) + if err != nil && r.FormValue("gid") != "" { + glog.V(0).Infof("register %s to %s parse gid %s: %v", fileId, path, r.FormValue("gid"), err) + writeJsonError(w, r, http.StatusInternalServerError, err) + return + } entry := &filer2.Entry{ FullPath: filer2.FullPath(path), Attr: filer2.Attr{ Mode: 0660, + Uid: uint32(uid), + Gid: uint32(gid), }, Chunks: []*filer_pb.FileChunk{{ FileId: fileId, From 7362de9a1895744a42931a8ccbdc5f8e4e2306b6 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Tue, 22 May 2018 03:26:38 -0700 Subject: [PATCH 16/83] weed mount can work well TODO: somehow filer url is returning empty content --- weed/filer2/embedded/embedded_store.go | 2 +- weed/filer2/filechunks.go | 15 +++ weed/filer2/filer.go | 5 +- weed/filer2/filer_structure.go | 2 +- weed/filer2/memdb/memdb_store.go | 9 +- weed/filesys/dir.go | 14 ++- weed/filesys/file.go | 113 +++++++++++------- weed/pb/filer.proto | 6 +- weed/pb/filer_pb/filer.pb.go | 154 ++++++++++++------------- weed/server/filer_grpc_server.go | 34 ++++-- 10 files changed, 207 insertions(+), 147 deletions(-) diff --git a/weed/filer2/embedded/embedded_store.go b/weed/filer2/embedded/embedded_store.go index d0cfac3f9..5cecbd25c 100644 --- a/weed/filer2/embedded/embedded_store.go +++ b/weed/filer2/embedded/embedded_store.go @@ -26,7 +26,7 @@ func (filer *EmbeddedStore) AddDirectoryLink(directory *filer2.Entry, delta int3 return nil } -func (filer *EmbeddedStore) SetFileChunks(fullpath filer2.FullPath, fileChunks []*filer_pb.FileChunk) (err error) { +func (filer *EmbeddedStore) UpdateEntry(entry *filer2.Entry) (err error) { return nil } diff --git a/weed/filer2/filechunks.go b/weed/filer2/filechunks.go index 714bdab9c..834e014d9 100644 --- a/weed/filer2/filechunks.go +++ b/weed/filer2/filechunks.go @@ -37,6 +37,21 @@ func CompactFileChunks(chunks []*filer_pb.FileChunk) (compacted, garbage []*file return } +func FindUnusedFileChunks(oldChunks, newChunks []*filer_pb.FileChunk) (unused []*filer_pb.FileChunk) { + + fileIds := make(map[string]bool) + for _, interval := range newChunks { + fileIds[interval.FileId] = true + } + for _, chunk := range oldChunks { + if found := fileIds[chunk.FileId]; !found { + unused = append(unused, chunk) + } + } + + return +} + func logPrintf(name string, visibles []*visibleInterval) { // return diff --git a/weed/filer2/filer.go b/weed/filer2/filer.go index facd447f8..6b83db0e6 100644 --- a/weed/filer2/filer.go +++ b/weed/filer2/filer.go @@ -8,7 +8,6 @@ import ( "path/filepath" "time" "os" - "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "github.com/chrislusf/seaweedfs/weed/glog" ) @@ -113,8 +112,8 @@ func (f *Filer) CreateEntry(entry *Entry) (error) { return nil } -func (f *Filer) SetFileChunks(p FullPath, chunks []*filer_pb.FileChunk) (err error) { - return f.store.SetFileChunks(p, chunks) +func (f *Filer) UpdateEntry(entry *Entry) (err error) { + return f.store.UpdateEntry(entry) } func (f *Filer) FindEntry(p FullPath) (found bool, entry *Entry, err error) { diff --git a/weed/filer2/filer_structure.go b/weed/filer2/filer_structure.go index 99ca9ce5a..5e5382d35 100644 --- a/weed/filer2/filer_structure.go +++ b/weed/filer2/filer_structure.go @@ -69,7 +69,7 @@ var ErrNotFound = errors.New("filer: no entry is found in filer store") type FilerStore interface { InsertEntry(*Entry) (error) - SetFileChunks(FullPath, []*filer_pb.FileChunk) (err error) + UpdateEntry(*Entry) (err error) FindEntry(FullPath) (found bool, entry *Entry, err error) DeleteEntry(FullPath) (fileEntry *Entry, err error) diff --git a/weed/filer2/memdb/memdb_store.go b/weed/filer2/memdb/memdb_store.go index 1e888cc45..ba26eb163 100644 --- a/weed/filer2/memdb/memdb_store.go +++ b/weed/filer2/memdb/memdb_store.go @@ -6,7 +6,6 @@ import ( "strings" "fmt" "time" - "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" ) type MemDbStore struct { @@ -29,16 +28,16 @@ func NewMemDbStore() (filer *MemDbStore) { func (filer *MemDbStore) InsertEntry(entry *filer2.Entry) (err error) { // println("inserting", entry.FullPath) + entry.Crtime = time.Now() filer.tree.ReplaceOrInsert(Entry{entry}) return nil } -func (filer *MemDbStore) SetFileChunks(fullpath filer2.FullPath, fileChunks []*filer_pb.FileChunk) (err error) { - found, entry, err := filer.FindEntry(fullpath) +func (filer *MemDbStore) UpdateEntry(entry *filer2.Entry) (err error) { + found, entry, err := filer.FindEntry(entry.FullPath) if !found { - return fmt.Errorf("No such file: %s", fullpath) + return fmt.Errorf("No such file: %s", entry.FullPath) } - entry.Chunks = fileChunks entry.Mtime = time.Now() filer.tree.ReplaceOrInsert(Entry{entry}) return nil diff --git a/weed/filesys/dir.go b/weed/filesys/dir.go index 90c95d4ce..3a0ca9b31 100644 --- a/weed/filesys/dir.go +++ b/weed/filesys/dir.go @@ -76,6 +76,16 @@ func (dir *Dir) Attr(context context.Context, attr *fuse.Attr) error { return nil } +func (dir *Dir) newFile(name string, chunks []*filer_pb.FileChunk) *File { + return &File{ + Name: name, + dir: dir, + wfs: dir.wfs, + // attributes: &filer_pb.FuseAttributes{}, + Chunks: chunks, + } +} + func (dir *Dir) Create(ctx context.Context, req *fuse.CreateRequest, resp *fuse.CreateResponse) (fs.Node, fs.Handle, error) { @@ -104,7 +114,7 @@ func (dir *Dir) Create(ctx context.Context, req *fuse.CreateRequest, }) if err == nil { - node := &File{Name: req.Name, dir: dir, wfs: dir.wfs} + node := dir.newFile(req.Name, nil) dir.NodeMap[req.Name] = node return node, node, nil } @@ -186,7 +196,7 @@ func (dir *Dir) Lookup(ctx context.Context, name string) (node fs.Node, err erro if entry.IsDirectory { node = &Dir{Path: path.Join(dir.Path, name), wfs: dir.wfs} } else { - node = &File{Chunks: entry.Chunks, Name: name, dir: dir, wfs: dir.wfs} + node = dir.newFile(name, entry.Chunks) } dir.NodeMap[name] = node return node, nil diff --git a/weed/filesys/file.go b/weed/filesys/file.go index 81dcba0d9..bbb014bd4 100644 --- a/weed/filesys/file.go +++ b/weed/filesys/file.go @@ -17,7 +17,7 @@ import ( ) var _ = fs.Node(&File{}) -// var _ = fs.NodeOpener(&File{}) +var _ = fs.NodeOpener(&File{}) var _ = fs.NodeFsyncer(&File{}) var _ = fs.Handle(&File{}) var _ = fs.HandleReadAller(&File{}) @@ -28,58 +28,63 @@ var _ = fs.HandleReleaser(&File{}) var _ = fs.NodeSetattrer(&File{}) type File struct { - Chunks []*filer_pb.FileChunk - Name string - dir *Dir - wfs *WFS + Chunks []*filer_pb.FileChunk + Name string + dir *Dir + wfs *WFS + isOpened bool + attributes *filer_pb.FuseAttributes } func (file *File) Attr(context context.Context, attr *fuse.Attr) error { - fullPath := filepath.Join(file.dir.Path, file.Name) - item := file.wfs.listDirectoryEntriesCache.Get(fullPath) - var attributes *filer_pb.FuseAttributes - if item != nil { - attributes = item.Value().(*filer_pb.FuseAttributes) - glog.V(1).Infof("read cached file %v attributes", file.Name) - } else { - err := file.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error { - - request := &filer_pb.GetEntryAttributesRequest{ - Name: file.Name, - ParentDir: file.dir.Path, - } - glog.V(1).Infof("read file size: %v", request) - resp, err := client.GetEntryAttributes(context, request) - if err != nil { - glog.V(0).Infof("read file attributes %v: %v", request, err) - return err - } + if !file.isOpened || file.attributes == nil { + fullPath := filepath.Join(file.dir.Path, file.Name) + item := file.wfs.listDirectoryEntriesCache.Get(fullPath) + if item != nil { + file.attributes = item.Value().(*filer_pb.FuseAttributes) + glog.V(1).Infof("read cached file %v attributes", file.Name) + } else { + err := file.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error { - attributes = resp.Attributes + request := &filer_pb.GetEntryAttributesRequest{ + Name: file.Name, + ParentDir: file.dir.Path, + } - return nil - }) + glog.V(1).Infof("read file size: %v", request) + resp, err := client.GetEntryAttributes(context, request) + if err != nil { + glog.V(0).Infof("read file attributes %v: %v", request, err) + return err + } - if err != nil { - return err + file.attributes = resp.Attributes + + return nil + }) + + if err != nil { + return err + } } } - attr.Mode = os.FileMode(attributes.FileMode) - attr.Size = attributes.FileSize - attr.Mtime = time.Unix(attributes.Mtime, 0) - attr.Gid = attributes.Gid - attr.Uid = attributes.Uid + attr.Mode = os.FileMode(file.attributes.FileMode) + attr.Size = filer2.TotalSize(file.Chunks) + attr.Mtime = time.Unix(file.attributes.Mtime, 0) + attr.Gid = file.attributes.Gid + attr.Uid = file.attributes.Uid return nil } -func (file *File) xOpen(ctx context.Context, req *fuse.OpenRequest, resp *fuse.OpenResponse) (fs.Handle, error) { +func (file *File) Open(ctx context.Context, req *fuse.OpenRequest, resp *fuse.OpenResponse) (fs.Handle, error) { fullPath := filepath.Join(file.dir.Path, file.Name) fmt.Printf("Open %v %+v\n", fullPath, req) + file.isOpened = true return file, nil @@ -89,10 +94,28 @@ func (file *File) Setattr(ctx context.Context, req *fuse.SetattrRequest, resp *f fullPath := filepath.Join(file.dir.Path, file.Name) fmt.Printf("Setattr %v %+v\n", fullPath, req) - if req.Valid.Size() && req.Size == 0 { - fmt.Printf("truncate %v \n", fullPath) - file.Chunks = nil - resp.Attr.Size = 0 + if req.Valid.Size() { + + if req.Size == 0 { + fmt.Printf("truncate %v \n", fullPath) + file.Chunks = nil + } + file.attributes.FileSize = req.Size + } + if req.Valid.Mode() { + file.attributes.FileMode = uint32(req.Mode) + } + + if req.Valid.Uid() { + file.attributes.Uid = req.Uid + } + + if req.Valid.Gid() { + file.attributes.Gid = req.Gid + } + + if req.Valid.Mtime() { + file.attributes.Mtime = req.Mtime.Unix() } return nil @@ -152,16 +175,17 @@ func (file *File) Flush(ctx context.Context, req *fuse.FlushRequest) error { err := file.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error { - request := &filer_pb.SetFileChunksRequest{ + request := &filer_pb.UpdateEntryRequest{ Directory: file.dir.Path, Entry: &filer_pb.Entry{ - Name: file.Name, - Chunks: file.Chunks, + Name: file.Name, + Attributes: file.attributes, + Chunks: file.Chunks, }, } glog.V(1).Infof("append chunks: %v", request) - if _, err := client.SetFileChunks(ctx, request); err != nil { + if _, err := client.UpdateEntry(ctx, request); err != nil { return fmt.Errorf("create file: %v", err) } @@ -224,7 +248,8 @@ func (file *File) Write(ctx context.Context, req *fuse.WriteRequest, resp *fuse. func (file *File) Release(ctx context.Context, req *fuse.ReleaseRequest) error { - // fmt.Printf("release file %+v\n", req) + fmt.Printf("release file %+v\n", req) + file.isOpened = false return nil } diff --git a/weed/pb/filer.proto b/weed/pb/filer.proto index 1f440d19c..e789f9d38 100644 --- a/weed/pb/filer.proto +++ b/weed/pb/filer.proto @@ -21,7 +21,7 @@ service SeaweedFiler { rpc CreateEntry (CreateEntryRequest) returns (CreateEntryResponse) { } - rpc SetFileChunks (SetFileChunksRequest) returns (SetFileChunksResponse) { + rpc UpdateEntry (UpdateEntryRequest) returns (UpdateEntryResponse) { } rpc DeleteEntry (DeleteEntryRequest) returns (DeleteEntryResponse) { @@ -121,9 +121,9 @@ message AssignVolumeResponse { int32 count = 4; } -message SetFileChunksRequest { +message UpdateEntryRequest { string directory = 1; Entry entry = 2; } -message SetFileChunksResponse { +message UpdateEntryResponse { } diff --git a/weed/pb/filer_pb/filer.pb.go b/weed/pb/filer_pb/filer.pb.go index a8a20ffe6..7ef1c294b 100644 --- a/weed/pb/filer_pb/filer.pb.go +++ b/weed/pb/filer_pb/filer.pb.go @@ -26,8 +26,8 @@ It has these top-level messages: DeleteEntryResponse AssignVolumeRequest AssignVolumeResponse - SetFileChunksRequest - SetFileChunksResponse + UpdateEntryRequest + UpdateEntryResponse */ package filer_pb @@ -475,37 +475,37 @@ func (m *AssignVolumeResponse) GetCount() int32 { return 0 } -type SetFileChunksRequest struct { +type UpdateEntryRequest struct { Directory string `protobuf:"bytes,1,opt,name=directory" json:"directory,omitempty"` Entry *Entry `protobuf:"bytes,2,opt,name=entry" json:"entry,omitempty"` } -func (m *SetFileChunksRequest) Reset() { *m = SetFileChunksRequest{} } -func (m *SetFileChunksRequest) String() string { return proto.CompactTextString(m) } -func (*SetFileChunksRequest) ProtoMessage() {} -func (*SetFileChunksRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{17} } +func (m *UpdateEntryRequest) Reset() { *m = UpdateEntryRequest{} } +func (m *UpdateEntryRequest) String() string { return proto.CompactTextString(m) } +func (*UpdateEntryRequest) ProtoMessage() {} +func (*UpdateEntryRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{17} } -func (m *SetFileChunksRequest) GetDirectory() string { +func (m *UpdateEntryRequest) GetDirectory() string { if m != nil { return m.Directory } return "" } -func (m *SetFileChunksRequest) GetEntry() *Entry { +func (m *UpdateEntryRequest) GetEntry() *Entry { if m != nil { return m.Entry } return nil } -type SetFileChunksResponse struct { +type UpdateEntryResponse struct { } -func (m *SetFileChunksResponse) Reset() { *m = SetFileChunksResponse{} } -func (m *SetFileChunksResponse) String() string { return proto.CompactTextString(m) } -func (*SetFileChunksResponse) ProtoMessage() {} -func (*SetFileChunksResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{18} } +func (m *UpdateEntryResponse) Reset() { *m = UpdateEntryResponse{} } +func (m *UpdateEntryResponse) String() string { return proto.CompactTextString(m) } +func (*UpdateEntryResponse) ProtoMessage() {} +func (*UpdateEntryResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{18} } func init() { proto.RegisterType((*LookupDirectoryEntryRequest)(nil), "filer_pb.LookupDirectoryEntryRequest") @@ -525,8 +525,8 @@ func init() { proto.RegisterType((*DeleteEntryResponse)(nil), "filer_pb.DeleteEntryResponse") proto.RegisterType((*AssignVolumeRequest)(nil), "filer_pb.AssignVolumeRequest") proto.RegisterType((*AssignVolumeResponse)(nil), "filer_pb.AssignVolumeResponse") - proto.RegisterType((*SetFileChunksRequest)(nil), "filer_pb.SetFileChunksRequest") - proto.RegisterType((*SetFileChunksResponse)(nil), "filer_pb.SetFileChunksResponse") + proto.RegisterType((*UpdateEntryRequest)(nil), "filer_pb.UpdateEntryRequest") + proto.RegisterType((*UpdateEntryResponse)(nil), "filer_pb.UpdateEntryResponse") } // Reference imports to suppress errors if they are not otherwise used. @@ -545,7 +545,7 @@ type SeaweedFilerClient interface { GetEntryAttributes(ctx context.Context, in *GetEntryAttributesRequest, opts ...grpc.CallOption) (*GetEntryAttributesResponse, error) GetFileContent(ctx context.Context, in *GetFileContentRequest, opts ...grpc.CallOption) (*GetFileContentResponse, error) CreateEntry(ctx context.Context, in *CreateEntryRequest, opts ...grpc.CallOption) (*CreateEntryResponse, error) - SetFileChunks(ctx context.Context, in *SetFileChunksRequest, opts ...grpc.CallOption) (*SetFileChunksResponse, error) + UpdateEntry(ctx context.Context, in *UpdateEntryRequest, opts ...grpc.CallOption) (*UpdateEntryResponse, error) DeleteEntry(ctx context.Context, in *DeleteEntryRequest, opts ...grpc.CallOption) (*DeleteEntryResponse, error) AssignVolume(ctx context.Context, in *AssignVolumeRequest, opts ...grpc.CallOption) (*AssignVolumeResponse, error) } @@ -603,9 +603,9 @@ func (c *seaweedFilerClient) CreateEntry(ctx context.Context, in *CreateEntryReq return out, nil } -func (c *seaweedFilerClient) SetFileChunks(ctx context.Context, in *SetFileChunksRequest, opts ...grpc.CallOption) (*SetFileChunksResponse, error) { - out := new(SetFileChunksResponse) - err := grpc.Invoke(ctx, "/filer_pb.SeaweedFiler/SetFileChunks", in, out, c.cc, opts...) +func (c *seaweedFilerClient) UpdateEntry(ctx context.Context, in *UpdateEntryRequest, opts ...grpc.CallOption) (*UpdateEntryResponse, error) { + out := new(UpdateEntryResponse) + err := grpc.Invoke(ctx, "/filer_pb.SeaweedFiler/UpdateEntry", in, out, c.cc, opts...) if err != nil { return nil, err } @@ -638,7 +638,7 @@ type SeaweedFilerServer interface { GetEntryAttributes(context.Context, *GetEntryAttributesRequest) (*GetEntryAttributesResponse, error) GetFileContent(context.Context, *GetFileContentRequest) (*GetFileContentResponse, error) CreateEntry(context.Context, *CreateEntryRequest) (*CreateEntryResponse, error) - SetFileChunks(context.Context, *SetFileChunksRequest) (*SetFileChunksResponse, error) + UpdateEntry(context.Context, *UpdateEntryRequest) (*UpdateEntryResponse, error) DeleteEntry(context.Context, *DeleteEntryRequest) (*DeleteEntryResponse, error) AssignVolume(context.Context, *AssignVolumeRequest) (*AssignVolumeResponse, error) } @@ -737,20 +737,20 @@ func _SeaweedFiler_CreateEntry_Handler(srv interface{}, ctx context.Context, dec return interceptor(ctx, in, info, handler) } -func _SeaweedFiler_SetFileChunks_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(SetFileChunksRequest) +func _SeaweedFiler_UpdateEntry_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(UpdateEntryRequest) if err := dec(in); err != nil { return nil, err } if interceptor == nil { - return srv.(SeaweedFilerServer).SetFileChunks(ctx, in) + return srv.(SeaweedFilerServer).UpdateEntry(ctx, in) } info := &grpc.UnaryServerInfo{ Server: srv, - FullMethod: "/filer_pb.SeaweedFiler/SetFileChunks", + FullMethod: "/filer_pb.SeaweedFiler/UpdateEntry", } handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(SeaweedFilerServer).SetFileChunks(ctx, req.(*SetFileChunksRequest)) + return srv.(SeaweedFilerServer).UpdateEntry(ctx, req.(*UpdateEntryRequest)) } return interceptor(ctx, in, info, handler) } @@ -816,8 +816,8 @@ var _SeaweedFiler_serviceDesc = grpc.ServiceDesc{ Handler: _SeaweedFiler_CreateEntry_Handler, }, { - MethodName: "SetFileChunks", - Handler: _SeaweedFiler_SetFileChunks_Handler, + MethodName: "UpdateEntry", + Handler: _SeaweedFiler_UpdateEntry_Handler, }, { MethodName: "DeleteEntry", @@ -835,53 +835,53 @@ var _SeaweedFiler_serviceDesc = grpc.ServiceDesc{ func init() { proto.RegisterFile("filer.proto", fileDescriptor0) } var fileDescriptor0 = []byte{ - // 765 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xac, 0x56, 0xdd, 0x4e, 0xdb, 0x4a, - 0x10, 0xc6, 0x38, 0x09, 0x64, 0x12, 0x38, 0x47, 0x93, 0x00, 0x3e, 0xe1, 0x2f, 0xc7, 0x2d, 0x15, - 0x55, 0x25, 0x54, 0xa5, 0x37, 0xbd, 0x2c, 0x02, 0x8a, 0x2a, 0x51, 0x21, 0x19, 0x81, 0x54, 0xf5, - 0x22, 0x4a, 0xec, 0x49, 0xba, 0xc2, 0xb1, 0x53, 0x7b, 0xdd, 0x8a, 0x5e, 0xf7, 0x55, 0xfa, 0x1e, - 0x7d, 0xb4, 0x6a, 0xd7, 0x1b, 0x7b, 0x4d, 0x1c, 0x5a, 0xa4, 0xde, 0x79, 0xe7, 0xef, 0xfb, 0x76, - 0x66, 0xbe, 0x4d, 0xa0, 0x31, 0x62, 0x3e, 0x45, 0x47, 0xd3, 0x28, 0xe4, 0x21, 0xae, 0xca, 0x43, - 0x7f, 0x3a, 0xb4, 0x2f, 0x61, 0xfb, 0x22, 0x0c, 0x6f, 0x93, 0xe9, 0x29, 0x8b, 0xc8, 0xe5, 0x61, - 0x74, 0x77, 0x16, 0xf0, 0xe8, 0xce, 0xa1, 0xcf, 0x09, 0xc5, 0x1c, 0x77, 0xa0, 0xee, 0xcd, 0x1c, - 0x96, 0xd1, 0x35, 0x0e, 0xeb, 0x4e, 0x6e, 0x40, 0x84, 0x4a, 0x30, 0x98, 0x90, 0xb5, 0x2c, 0x1d, - 0xf2, 0xdb, 0x3e, 0x83, 0x9d, 0xf2, 0x82, 0xf1, 0x34, 0x0c, 0x62, 0xc2, 0x03, 0xa8, 0x92, 0x30, - 0xc8, 0x6a, 0x8d, 0xde, 0x3f, 0x47, 0x33, 0x2a, 0x47, 0x69, 0x5c, 0xea, 0xb5, 0x7b, 0x80, 0x17, - 0x2c, 0xe6, 0xc2, 0xc6, 0x28, 0xfe, 0x23, 0x3a, 0xf6, 0x1b, 0x68, 0x15, 0x72, 0x14, 0xe2, 0x73, - 0x58, 0xa1, 0xd4, 0x64, 0x19, 0x5d, 0xb3, 0x0c, 0x73, 0xe6, 0xb7, 0x7f, 0x18, 0x50, 0x95, 0xa6, - 0xec, 0x6a, 0x46, 0x7e, 0x35, 0xfc, 0x1f, 0x9a, 0x2c, 0xee, 0xe7, 0x04, 0xc4, 0xb5, 0x57, 0x9d, - 0x06, 0x8b, 0xb3, 0xab, 0xe2, 0x0b, 0xa8, 0xb9, 0x9f, 0x92, 0xe0, 0x36, 0xb6, 0x4c, 0x09, 0xd5, - 0xca, 0xa1, 0xde, 0x32, 0x9f, 0x4e, 0x84, 0xcf, 0x51, 0x21, 0xf8, 0x1a, 0x60, 0xc0, 0x79, 0xc4, - 0x86, 0x09, 0xa7, 0xd8, 0xaa, 0xc8, 0x7e, 0x58, 0x5a, 0x42, 0x12, 0xd3, 0x71, 0xe6, 0x77, 0xb4, - 0x58, 0x7b, 0x04, 0xf5, 0xac, 0x1c, 0x6e, 0xc1, 0x8a, 0xc8, 0xe9, 0x33, 0x4f, 0xb1, 0xad, 0x89, - 0xe3, 0x3b, 0x0f, 0x37, 0xa1, 0x16, 0x8e, 0x46, 0x31, 0x71, 0xc9, 0xd4, 0x74, 0xd4, 0x49, 0xdc, - 0x2d, 0x66, 0xdf, 0xc8, 0x32, 0xbb, 0xc6, 0x61, 0xc5, 0x91, 0xdf, 0xd8, 0x86, 0xea, 0x84, 0xb3, - 0x09, 0x49, 0x1a, 0xa6, 0x93, 0x1e, 0xec, 0xef, 0x06, 0xac, 0x17, 0x69, 0xe0, 0x36, 0xd4, 0x25, - 0x9a, 0xac, 0x60, 0xc8, 0x0a, 0x72, 0x9b, 0xae, 0x0a, 0x55, 0x96, 0xb5, 0x2a, 0x59, 0xca, 0x24, - 0xf4, 0x52, 0xd0, 0xb5, 0x34, 0xe5, 0x7d, 0xe8, 0x11, 0xfe, 0x0b, 0x66, 0xc2, 0x3c, 0x09, 0xbb, - 0xe6, 0x88, 0x4f, 0x61, 0x19, 0x33, 0xcf, 0xaa, 0xa6, 0x96, 0x31, 0xf3, 0xec, 0x31, 0xfc, 0x77, - 0x4e, 0x72, 0xae, 0x77, 0x5a, 0x43, 0xd4, 0x4e, 0x94, 0x4d, 0x6a, 0x17, 0x60, 0x3a, 0x88, 0x28, - 0xe0, 0x62, 0x5a, 0x6a, 0x3d, 0xeb, 0xa9, 0xe5, 0x94, 0x45, 0x7a, 0xc7, 0x4c, 0xbd, 0x63, 0xf6, - 0x0d, 0x74, 0xca, 0x80, 0xd4, 0x22, 0x15, 0xe7, 0x65, 0x3c, 0x62, 0x5e, 0x2f, 0x61, 0xe3, 0x9c, - 0xb8, 0x1c, 0x59, 0x18, 0x70, 0x0a, 0xf8, 0x8c, 0xfc, 0xa2, 0xd9, 0xd9, 0x3d, 0xd8, 0xbc, 0x9f, - 0xa1, 0x58, 0x58, 0xb0, 0xe2, 0xa6, 0x26, 0x99, 0xd2, 0x74, 0x66, 0x47, 0xfb, 0x03, 0xe0, 0x49, - 0x44, 0x03, 0x4e, 0x8f, 0x90, 0x70, 0x26, 0xc7, 0xe5, 0x07, 0xe5, 0xb8, 0x01, 0xad, 0x42, 0xe9, - 0x94, 0x8b, 0xcd, 0x00, 0x4f, 0xc9, 0xa7, 0x47, 0x21, 0x96, 0x3c, 0x1a, 0x73, 0xca, 0x32, 0xe7, - 0x94, 0x25, 0x18, 0x14, 0xa0, 0x14, 0x83, 0x09, 0xb4, 0x8e, 0xe3, 0x98, 0x8d, 0x83, 0x9b, 0xd0, - 0x4f, 0x26, 0x34, 0xa3, 0xd0, 0x86, 0xaa, 0x1b, 0x26, 0xaa, 0x45, 0x55, 0x27, 0x3d, 0xe0, 0x1e, - 0x80, 0x1b, 0xfa, 0x3e, 0xb9, 0x9c, 0x85, 0x81, 0x22, 0xa0, 0x59, 0xb0, 0x0b, 0x8d, 0x88, 0xa6, - 0x3e, 0x73, 0x07, 0x32, 0x20, 0xdd, 0x0d, 0xdd, 0x64, 0x7f, 0x81, 0x76, 0x11, 0x4e, 0x0d, 0x65, - 0xa1, 0x06, 0xc5, 0x7a, 0x47, 0xbe, 0xc2, 0x12, 0x9f, 0x72, 0x37, 0x93, 0xa1, 0xcf, 0xdc, 0xbe, - 0x70, 0x98, 0x6a, 0x37, 0xa5, 0xe5, 0x3a, 0xf2, 0x73, 0xe6, 0x15, 0x8d, 0xb9, 0xfd, 0x11, 0xda, - 0x57, 0x6a, 0x1d, 0xe4, 0xdb, 0xf1, 0x57, 0x87, 0xbb, 0x05, 0x1b, 0xf7, 0x8a, 0xa7, 0xb7, 0xea, - 0xfd, 0xac, 0x42, 0xf3, 0x8a, 0x06, 0x5f, 0x89, 0x3c, 0xe1, 0x8d, 0x70, 0x0c, 0xed, 0xb2, 0xc7, - 0x1d, 0x0f, 0xf2, 0xca, 0x0f, 0xfc, 0x9a, 0x74, 0x9e, 0xfd, 0x2e, 0x4c, 0x0d, 0x75, 0x09, 0x2f, - 0xa0, 0xa1, 0x3d, 0xe5, 0xb8, 0xa3, 0x25, 0xce, 0xfd, 0x2a, 0x74, 0x76, 0x17, 0x78, 0xb3, 0x6a, - 0x03, 0xc0, 0x79, 0x59, 0xe3, 0x93, 0x3c, 0x6d, 0xe1, 0xeb, 0xd2, 0x79, 0xfa, 0x70, 0x50, 0x06, - 0x71, 0x0d, 0xeb, 0x45, 0xbd, 0xe2, 0x7e, 0x21, 0x73, 0x5e, 0xfb, 0x9d, 0xee, 0xe2, 0x00, 0xbd, - 0x0f, 0x9a, 0xee, 0xf4, 0x3e, 0xcc, 0x2b, 0x5d, 0xef, 0x43, 0x99, 0x58, 0x97, 0xd0, 0x81, 0xb5, - 0xc2, 0xa0, 0x71, 0x2f, 0xcf, 0x28, 0x5b, 0xaf, 0xce, 0xfe, 0x42, 0xbf, 0xce, 0x50, 0xd3, 0xa5, - 0xce, 0x70, 0xfe, 0x65, 0xd0, 0x19, 0x96, 0x89, 0x79, 0x09, 0x2f, 0xa1, 0xa9, 0xeb, 0x0b, 0xb5, - 0x84, 0x12, 0x99, 0x77, 0xf6, 0x16, 0xb9, 0x67, 0x05, 0x87, 0x35, 0xf9, 0x87, 0xe7, 0xd5, 0xaf, - 0x00, 0x00, 0x00, 0xff, 0xff, 0xc7, 0x51, 0xe7, 0x2b, 0xff, 0x08, 0x00, 0x00, + // 754 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xb4, 0x56, 0xdd, 0x6e, 0xd3, 0x4a, + 0x10, 0xae, 0xeb, 0x24, 0x6d, 0x26, 0x69, 0xcf, 0xd1, 0x26, 0xed, 0xf1, 0x49, 0x7f, 0x08, 0x86, + 0xa2, 0x22, 0xa4, 0x0a, 0x85, 0x1b, 0x2e, 0xa9, 0xda, 0x52, 0x21, 0x15, 0x55, 0x72, 0xd5, 0x4a, + 0x5c, 0x45, 0x89, 0x3d, 0x09, 0xab, 0x3a, 0x76, 0xf0, 0xae, 0x41, 0xe5, 0x9a, 0x57, 0xe1, 0x25, + 0x78, 0x3a, 0xb4, 0x3f, 0xb1, 0xd7, 0xd8, 0x29, 0xf4, 0x82, 0x3b, 0xef, 0xfc, 0x7c, 0xf3, 0xed, + 0xcc, 0x7c, 0x9b, 0x40, 0x6b, 0x42, 0x43, 0x4c, 0x8e, 0xe6, 0x49, 0xcc, 0x63, 0xb2, 0x2e, 0x0f, + 0xc3, 0xf9, 0xd8, 0xbd, 0x84, 0x9d, 0x8b, 0x38, 0xbe, 0x4d, 0xe7, 0xa7, 0x34, 0x41, 0x9f, 0xc7, + 0xc9, 0xdd, 0x59, 0xc4, 0x93, 0x3b, 0x0f, 0x3f, 0xa5, 0xc8, 0x38, 0xd9, 0x85, 0x66, 0xb0, 0x70, + 0x38, 0x56, 0xdf, 0x3a, 0x6c, 0x7a, 0xb9, 0x81, 0x10, 0xa8, 0x45, 0xa3, 0x19, 0x3a, 0xab, 0xd2, + 0x21, 0xbf, 0xdd, 0x33, 0xd8, 0xad, 0x06, 0x64, 0xf3, 0x38, 0x62, 0x48, 0x0e, 0xa0, 0x8e, 0xc2, + 0x20, 0xd1, 0x5a, 0x83, 0x7f, 0x8e, 0x16, 0x54, 0x8e, 0x54, 0x9c, 0xf2, 0xba, 0x03, 0x20, 0x17, + 0x94, 0x71, 0x61, 0xa3, 0xc8, 0xfe, 0x88, 0x8e, 0xfb, 0x06, 0x3a, 0x85, 0x1c, 0x5d, 0xf1, 0x39, + 0xac, 0xa1, 0x32, 0x39, 0x56, 0xdf, 0xae, 0xaa, 0xb9, 0xf0, 0xbb, 0xdf, 0x2d, 0xa8, 0x4b, 0x53, + 0x76, 0x35, 0x2b, 0xbf, 0x1a, 0x79, 0x0c, 0x6d, 0xca, 0x86, 0x39, 0x01, 0x71, 0xed, 0x75, 0xaf, + 0x45, 0x59, 0x76, 0x55, 0xf2, 0x02, 0x1a, 0xfe, 0xc7, 0x34, 0xba, 0x65, 0x8e, 0x2d, 0x4b, 0x75, + 0xf2, 0x52, 0x6f, 0x69, 0x88, 0x27, 0xc2, 0xe7, 0xe9, 0x10, 0xf2, 0x1a, 0x60, 0xc4, 0x79, 0x42, + 0xc7, 0x29, 0x47, 0xe6, 0xd4, 0x64, 0x3f, 0x1c, 0x23, 0x21, 0x65, 0x78, 0x9c, 0xf9, 0x3d, 0x23, + 0xd6, 0x9d, 0x40, 0x33, 0x83, 0x23, 0xff, 0xc1, 0x9a, 0xc8, 0x19, 0xd2, 0x40, 0xb3, 0x6d, 0x88, + 0xe3, 0xbb, 0x80, 0x6c, 0x43, 0x23, 0x9e, 0x4c, 0x18, 0x72, 0xc9, 0xd4, 0xf6, 0xf4, 0x49, 0xdc, + 0x8d, 0xd1, 0xaf, 0xe8, 0xd8, 0x7d, 0xeb, 0xb0, 0xe6, 0xc9, 0x6f, 0xd2, 0x85, 0xfa, 0x8c, 0xd3, + 0x19, 0x4a, 0x1a, 0xb6, 0xa7, 0x0e, 0xee, 0x37, 0x0b, 0x36, 0x8b, 0x34, 0xc8, 0x0e, 0x34, 0x65, + 0x35, 0x89, 0x60, 0x49, 0x04, 0xb9, 0x4d, 0x57, 0x05, 0x94, 0x55, 0x03, 0x25, 0x4b, 0x99, 0xc5, + 0x81, 0x2a, 0xba, 0xa1, 0x52, 0xde, 0xc7, 0x01, 0x92, 0x7f, 0xc1, 0x4e, 0x69, 0x20, 0xcb, 0x6e, + 0x78, 0xe2, 0x53, 0x58, 0xa6, 0x34, 0x70, 0xea, 0xca, 0x32, 0xa5, 0x81, 0x3b, 0x85, 0xff, 0xcf, + 0x51, 0xce, 0xf5, 0xce, 0x68, 0x88, 0xde, 0x89, 0xaa, 0x49, 0xed, 0x01, 0xcc, 0x47, 0x09, 0x46, + 0x5c, 0x4c, 0x4b, 0xaf, 0x67, 0x53, 0x59, 0x4e, 0x69, 0x62, 0x76, 0xcc, 0x36, 0x3b, 0xe6, 0xde, + 0x40, 0xaf, 0xaa, 0x90, 0x5e, 0xa4, 0xe2, 0xbc, 0xac, 0x07, 0xcc, 0xeb, 0x25, 0x6c, 0x9d, 0x23, + 0x97, 0x23, 0x8b, 0x23, 0x8e, 0x11, 0x5f, 0x90, 0x5f, 0x36, 0x3b, 0x77, 0x00, 0xdb, 0xbf, 0x66, + 0x68, 0x16, 0x0e, 0xac, 0xf9, 0xca, 0x24, 0x53, 0xda, 0xde, 0xe2, 0xe8, 0x7e, 0x00, 0x72, 0x92, + 0xe0, 0x88, 0xe3, 0x03, 0x24, 0x9c, 0xc9, 0x71, 0xf5, 0x5e, 0x39, 0x6e, 0x41, 0xa7, 0x00, 0xad, + 0xb8, 0xb8, 0x14, 0xc8, 0x29, 0x86, 0xf8, 0xa0, 0x8a, 0x15, 0x8f, 0x46, 0x49, 0x59, 0x76, 0x49, + 0x59, 0x82, 0x41, 0xa1, 0x94, 0x66, 0x30, 0x83, 0xce, 0x31, 0x63, 0x74, 0x1a, 0xdd, 0xc4, 0x61, + 0x3a, 0xc3, 0x05, 0x85, 0x2e, 0xd4, 0xfd, 0x38, 0xd5, 0x2d, 0xaa, 0x7b, 0xea, 0x40, 0xf6, 0x01, + 0xfc, 0x38, 0x0c, 0xd1, 0xe7, 0x34, 0x8e, 0x34, 0x01, 0xc3, 0x42, 0xfa, 0xd0, 0x4a, 0x70, 0x1e, + 0x52, 0x7f, 0x24, 0x03, 0xd4, 0x6e, 0x98, 0x26, 0xf7, 0x33, 0x74, 0x8b, 0xe5, 0xf4, 0x50, 0x96, + 0x6a, 0x50, 0xac, 0x77, 0x12, 0xea, 0x5a, 0xe2, 0x53, 0xee, 0x66, 0x3a, 0x0e, 0xa9, 0x3f, 0x14, + 0x0e, 0x5b, 0xef, 0xa6, 0xb4, 0x5c, 0x27, 0x61, 0xce, 0xbc, 0x66, 0x30, 0x17, 0xa3, 0xbd, 0x9e, + 0x07, 0x7f, 0x6b, 0xb4, 0x05, 0x68, 0x75, 0xa3, 0xc1, 0x8f, 0x3a, 0xb4, 0xaf, 0x70, 0xf4, 0x05, + 0x31, 0x10, 0x5b, 0x98, 0x90, 0x29, 0x74, 0xab, 0x1e, 0x76, 0x72, 0x90, 0xe3, 0xde, 0xf3, 0x4b, + 0xd2, 0x7b, 0xf6, 0xbb, 0x30, 0x3d, 0xd0, 0x15, 0x72, 0x01, 0x2d, 0xe3, 0x19, 0x27, 0xbb, 0x46, + 0x62, 0xe9, 0x17, 0xa1, 0xb7, 0xb7, 0xc4, 0x9b, 0xa1, 0x8d, 0x80, 0x94, 0x25, 0x4d, 0x9e, 0xe4, + 0x69, 0x4b, 0x5f, 0x96, 0xde, 0xd3, 0xfb, 0x83, 0xb2, 0x12, 0xd7, 0xb0, 0x59, 0xd4, 0x2a, 0x79, + 0x54, 0xc8, 0x2c, 0xeb, 0xbe, 0xd7, 0x5f, 0x1e, 0x60, 0xf6, 0xc1, 0xd0, 0x9c, 0xd9, 0x87, 0xb2, + 0xca, 0xcd, 0x3e, 0x54, 0x09, 0x55, 0xa2, 0x19, 0x63, 0x36, 0xd1, 0xca, 0x8b, 0x65, 0xa2, 0x55, + 0xec, 0x86, 0x42, 0x33, 0xd4, 0x68, 0xa2, 0x95, 0xdf, 0x03, 0x13, 0xad, 0x4a, 0xc2, 0x2b, 0xe4, + 0x12, 0xda, 0xa6, 0xaa, 0x88, 0x91, 0x50, 0x21, 0xee, 0xde, 0xfe, 0x32, 0xf7, 0x02, 0x70, 0xdc, + 0x90, 0x7f, 0x73, 0x5e, 0xfd, 0x0c, 0x00, 0x00, 0xff, 0xff, 0x9c, 0xf1, 0x24, 0x43, 0xf5, 0x08, + 0x00, 0x00, } diff --git a/weed/server/filer_grpc_server.go b/weed/server/filer_grpc_server.go index 3f05d777b..8d643ed2d 100644 --- a/weed/server/filer_grpc_server.go +++ b/weed/server/filer_grpc_server.go @@ -120,34 +120,46 @@ func (fs *FilerServer) CreateEntry(ctx context.Context, req *filer_pb.CreateEntr return &filer_pb.CreateEntryResponse{}, err } -func (fs *FilerServer) SetFileChunks(ctx context.Context, req *filer_pb.SetFileChunksRequest) (*filer_pb.SetFileChunksResponse, error) { +func (fs *FilerServer) UpdateEntry(ctx context.Context, req *filer_pb.UpdateEntryRequest) (*filer_pb.UpdateEntryResponse, error) { fullpath := filepath.Join(req.Directory, req.Entry.Name) found, entry, err := fs.filer.FindEntry(filer2.FullPath(fullpath)) if err != nil { - return &filer_pb.SetFileChunksResponse{}, err + return &filer_pb.UpdateEntryResponse{}, err } if !found { - return &filer_pb.SetFileChunksResponse{}, fmt.Errorf("file not found: %s", fullpath) + return &filer_pb.UpdateEntryResponse{}, fmt.Errorf("file not found: %s", fullpath) } - chunks := append(entry.Chunks, req.Entry.Chunks...) + // remove old chunks if not included in the new ones + unusedChunks := filer2.FindUnusedFileChunks(entry.Chunks, req.Entry.Chunks) - chunks, garbages := filer2.CompactFileChunks(chunks) + chunks, garbages := filer2.CompactFileChunks(req.Entry.Chunks) - err = fs.filer.SetFileChunks( - filer2.FullPath(fullpath), - chunks, - ) + err = fs.filer.UpdateEntry(&filer2.Entry{ + FullPath: filer2.FullPath(filepath.Join(req.Directory, req.Entry.Name)), + Attr: filer2.Attr{ + Mtime: time.Unix(req.Entry.Attributes.Mtime, 0), + Crtime: time.Unix(req.Entry.Attributes.Mtime, 0), + Mode: os.FileMode(req.Entry.Attributes.FileMode), + Uid: req.Entry.Attributes.Uid, + Gid: req.Entry.Attributes.Gid, + }, + Chunks: chunks, + }) if err == nil { - for _, garbage := range garbages { + for _, garbage := range unusedChunks { glog.V(0).Infof("deleting %s old chunk: %v, [%d, %d)", fullpath, garbage.FileId, garbage.Offset, garbage.Offset+int64(garbage.Size)) operation.DeleteFile(fs.master, garbage.FileId, fs.jwt(garbage.FileId)) } + for _, garbage := range garbages { + glog.V(0).Infof("deleting %s garbage chunk: %v, [%d, %d)", fullpath, garbage.FileId, garbage.Offset, garbage.Offset+int64(garbage.Size)) + operation.DeleteFile(fs.master, garbage.FileId, fs.jwt(garbage.FileId)) + } } - return &filer_pb.SetFileChunksResponse{}, err + return &filer_pb.UpdateEntryResponse{}, err } func (fs *FilerServer) DeleteEntry(ctx context.Context, req *filer_pb.DeleteEntryRequest) (resp *filer_pb.DeleteEntryResponse, err error) { From 873868cc10a638cf30387e0b8f11d91f3509ce34 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Tue, 22 May 2018 04:31:44 -0700 Subject: [PATCH 17/83] not working now need to add file handler --- weed/filer2/embedded/embedded_store.go | 1 - weed/filesys/dir.go | 6 +- weed/filesys/file.go | 17 ++-- weed/pb/filer.proto | 1 + weed/pb/filer_pb/filer.pb.go | 104 +++++++++++++------------ weed/server/filer_grpc_server.go | 17 ++-- 6 files changed, 80 insertions(+), 66 deletions(-) diff --git a/weed/filer2/embedded/embedded_store.go b/weed/filer2/embedded/embedded_store.go index 5cecbd25c..bca182f36 100644 --- a/weed/filer2/embedded/embedded_store.go +++ b/weed/filer2/embedded/embedded_store.go @@ -3,7 +3,6 @@ package embedded import ( "github.com/syndtr/goleveldb/leveldb" "github.com/chrislusf/seaweedfs/weed/filer2" - "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" ) type EmbeddedStore struct { diff --git a/weed/filesys/dir.go b/weed/filesys/dir.go index 3a0ca9b31..51b617689 100644 --- a/weed/filesys/dir.go +++ b/weed/filesys/dir.go @@ -181,7 +181,7 @@ func (dir *Dir) Lookup(ctx context.Context, name string) (node fs.Node, err erro Name: name, } - glog.V(1).Infof("lookup directory entry: %v", request) + glog.V(4).Infof("lookup directory entry: %v", request) resp, err := client.LookupDirectoryEntry(ctx, request) if err != nil { return err @@ -213,7 +213,7 @@ func (dir *Dir) ReadDirAll(ctx context.Context) (ret []fuse.Dirent, err error) { Directory: dir.Path, } - glog.V(1).Infof("read directory: %v", request) + glog.V(4).Infof("read directory: %v", request) resp, err := client.ListEntries(ctx, request) if err != nil { return err @@ -226,7 +226,7 @@ func (dir *Dir) ReadDirAll(ctx context.Context) (ret []fuse.Dirent, err error) { } else { dirent := fuse.Dirent{Name: entry.Name, Type: fuse.DT_File} ret = append(ret, dirent) - dir.wfs.listDirectoryEntriesCache.Set(dir.Path+"/"+entry.Name, entry.Attributes, 300*time.Millisecond) + dir.wfs.listDirectoryEntriesCache.Set(dir.Path+"/"+entry.Name, entry, 300*time.Millisecond) } } diff --git a/weed/filesys/file.go b/weed/filesys/file.go index bbb014bd4..9604d6fb0 100644 --- a/weed/filesys/file.go +++ b/weed/filesys/file.go @@ -38,11 +38,13 @@ type File struct { func (file *File) Attr(context context.Context, attr *fuse.Attr) error { - if !file.isOpened || file.attributes == nil { + if !file.isOpened { fullPath := filepath.Join(file.dir.Path, file.Name) item := file.wfs.listDirectoryEntriesCache.Get(fullPath) if item != nil { - file.attributes = item.Value().(*filer_pb.FuseAttributes) + entry := item.Value().(*filer_pb.Entry) + file.Chunks = entry.Chunks + file.attributes = entry.Attributes glog.V(1).Infof("read cached file %v attributes", file.Name) } else { err := file.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error { @@ -52,14 +54,15 @@ func (file *File) Attr(context context.Context, attr *fuse.Attr) error { ParentDir: file.dir.Path, } - glog.V(1).Infof("read file size: %v", request) + glog.V(1).Infof("read file: %v", request) resp, err := client.GetEntryAttributes(context, request) if err != nil { - glog.V(0).Infof("read file attributes %v: %v", request, err) + glog.V(0).Infof("read file %v: %v", request, err) return err } file.attributes = resp.Attributes + file.Chunks = resp.Chunks return nil }) @@ -169,7 +172,7 @@ func (file *File) Flush(ctx context.Context, req *fuse.FlushRequest) error { glog.V(3).Infof("file flush %v", req) if len(file.Chunks) == 0 { - glog.V(2).Infof("file flush skipping empty %v", req) + glog.V(2).Infof("%x file %s/%s flush skipping empty: %v", file, file.dir.Path, file.Name, req) return nil } @@ -184,9 +187,9 @@ func (file *File) Flush(ctx context.Context, req *fuse.FlushRequest) error { }, } - glog.V(1).Infof("append chunks: %v", request) + glog.V(1).Infof("%s/%s set chunks: %v", file.dir.Path, file.Name, len(file.Chunks)) if _, err := client.UpdateEntry(ctx, request); err != nil { - return fmt.Errorf("create file: %v", err) + return fmt.Errorf("update file: %v", err) } return nil diff --git a/weed/pb/filer.proto b/weed/pb/filer.proto index e789f9d38..b9c3e87f7 100644 --- a/weed/pb/filer.proto +++ b/weed/pb/filer.proto @@ -81,6 +81,7 @@ message GetEntryAttributesRequest { message GetEntryAttributesResponse { FuseAttributes attributes = 1; + repeated FileChunk chunks = 2; } message GetFileContentRequest { diff --git a/weed/pb/filer_pb/filer.pb.go b/weed/pb/filer_pb/filer.pb.go index 7ef1c294b..b1f0a3e0d 100644 --- a/weed/pb/filer_pb/filer.pb.go +++ b/weed/pb/filer_pb/filer.pb.go @@ -285,6 +285,7 @@ func (m *GetEntryAttributesRequest) GetFileId() string { type GetEntryAttributesResponse struct { Attributes *FuseAttributes `protobuf:"bytes,1,opt,name=attributes" json:"attributes,omitempty"` + Chunks []*FileChunk `protobuf:"bytes,2,rep,name=chunks" json:"chunks,omitempty"` } func (m *GetEntryAttributesResponse) Reset() { *m = GetEntryAttributesResponse{} } @@ -299,6 +300,13 @@ func (m *GetEntryAttributesResponse) GetAttributes() *FuseAttributes { return nil } +func (m *GetEntryAttributesResponse) GetChunks() []*FileChunk { + if m != nil { + return m.Chunks + } + return nil +} + type GetFileContentRequest struct { FileId string `protobuf:"bytes,1,opt,name=file_id,json=fileId" json:"file_id,omitempty"` } @@ -835,53 +843,53 @@ var _SeaweedFiler_serviceDesc = grpc.ServiceDesc{ func init() { proto.RegisterFile("filer.proto", fileDescriptor0) } var fileDescriptor0 = []byte{ - // 754 bytes of a gzipped FileDescriptorProto + // 763 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xb4, 0x56, 0xdd, 0x6e, 0xd3, 0x4a, - 0x10, 0xae, 0xeb, 0x24, 0x6d, 0x26, 0x69, 0xcf, 0xd1, 0x26, 0xed, 0xf1, 0x49, 0x7f, 0x08, 0x86, - 0xa2, 0x22, 0xa4, 0x0a, 0x85, 0x1b, 0x2e, 0xa9, 0xda, 0x52, 0x21, 0x15, 0x55, 0x72, 0xd5, 0x4a, - 0x5c, 0x45, 0x89, 0x3d, 0x09, 0xab, 0x3a, 0x76, 0xf0, 0xae, 0x41, 0xe5, 0x9a, 0x57, 0xe1, 0x25, - 0x78, 0x3a, 0xb4, 0x3f, 0xb1, 0xd7, 0xd8, 0x29, 0xf4, 0x82, 0x3b, 0xef, 0xfc, 0x7c, 0xf3, 0xed, - 0xcc, 0x7c, 0x9b, 0x40, 0x6b, 0x42, 0x43, 0x4c, 0x8e, 0xe6, 0x49, 0xcc, 0x63, 0xb2, 0x2e, 0x0f, - 0xc3, 0xf9, 0xd8, 0xbd, 0x84, 0x9d, 0x8b, 0x38, 0xbe, 0x4d, 0xe7, 0xa7, 0x34, 0x41, 0x9f, 0xc7, - 0xc9, 0xdd, 0x59, 0xc4, 0x93, 0x3b, 0x0f, 0x3f, 0xa5, 0xc8, 0x38, 0xd9, 0x85, 0x66, 0xb0, 0x70, - 0x38, 0x56, 0xdf, 0x3a, 0x6c, 0x7a, 0xb9, 0x81, 0x10, 0xa8, 0x45, 0xa3, 0x19, 0x3a, 0xab, 0xd2, - 0x21, 0xbf, 0xdd, 0x33, 0xd8, 0xad, 0x06, 0x64, 0xf3, 0x38, 0x62, 0x48, 0x0e, 0xa0, 0x8e, 0xc2, - 0x20, 0xd1, 0x5a, 0x83, 0x7f, 0x8e, 0x16, 0x54, 0x8e, 0x54, 0x9c, 0xf2, 0xba, 0x03, 0x20, 0x17, - 0x94, 0x71, 0x61, 0xa3, 0xc8, 0xfe, 0x88, 0x8e, 0xfb, 0x06, 0x3a, 0x85, 0x1c, 0x5d, 0xf1, 0x39, - 0xac, 0xa1, 0x32, 0x39, 0x56, 0xdf, 0xae, 0xaa, 0xb9, 0xf0, 0xbb, 0xdf, 0x2d, 0xa8, 0x4b, 0x53, - 0x76, 0x35, 0x2b, 0xbf, 0x1a, 0x79, 0x0c, 0x6d, 0xca, 0x86, 0x39, 0x01, 0x71, 0xed, 0x75, 0xaf, - 0x45, 0x59, 0x76, 0x55, 0xf2, 0x02, 0x1a, 0xfe, 0xc7, 0x34, 0xba, 0x65, 0x8e, 0x2d, 0x4b, 0x75, - 0xf2, 0x52, 0x6f, 0x69, 0x88, 0x27, 0xc2, 0xe7, 0xe9, 0x10, 0xf2, 0x1a, 0x60, 0xc4, 0x79, 0x42, - 0xc7, 0x29, 0x47, 0xe6, 0xd4, 0x64, 0x3f, 0x1c, 0x23, 0x21, 0x65, 0x78, 0x9c, 0xf9, 0x3d, 0x23, - 0xd6, 0x9d, 0x40, 0x33, 0x83, 0x23, 0xff, 0xc1, 0x9a, 0xc8, 0x19, 0xd2, 0x40, 0xb3, 0x6d, 0x88, - 0xe3, 0xbb, 0x80, 0x6c, 0x43, 0x23, 0x9e, 0x4c, 0x18, 0x72, 0xc9, 0xd4, 0xf6, 0xf4, 0x49, 0xdc, - 0x8d, 0xd1, 0xaf, 0xe8, 0xd8, 0x7d, 0xeb, 0xb0, 0xe6, 0xc9, 0x6f, 0xd2, 0x85, 0xfa, 0x8c, 0xd3, - 0x19, 0x4a, 0x1a, 0xb6, 0xa7, 0x0e, 0xee, 0x37, 0x0b, 0x36, 0x8b, 0x34, 0xc8, 0x0e, 0x34, 0x65, - 0x35, 0x89, 0x60, 0x49, 0x04, 0xb9, 0x4d, 0x57, 0x05, 0x94, 0x55, 0x03, 0x25, 0x4b, 0x99, 0xc5, - 0x81, 0x2a, 0xba, 0xa1, 0x52, 0xde, 0xc7, 0x01, 0x92, 0x7f, 0xc1, 0x4e, 0x69, 0x20, 0xcb, 0x6e, - 0x78, 0xe2, 0x53, 0x58, 0xa6, 0x34, 0x70, 0xea, 0xca, 0x32, 0xa5, 0x81, 0x3b, 0x85, 0xff, 0xcf, - 0x51, 0xce, 0xf5, 0xce, 0x68, 0x88, 0xde, 0x89, 0xaa, 0x49, 0xed, 0x01, 0xcc, 0x47, 0x09, 0x46, - 0x5c, 0x4c, 0x4b, 0xaf, 0x67, 0x53, 0x59, 0x4e, 0x69, 0x62, 0x76, 0xcc, 0x36, 0x3b, 0xe6, 0xde, - 0x40, 0xaf, 0xaa, 0x90, 0x5e, 0xa4, 0xe2, 0xbc, 0xac, 0x07, 0xcc, 0xeb, 0x25, 0x6c, 0x9d, 0x23, - 0x97, 0x23, 0x8b, 0x23, 0x8e, 0x11, 0x5f, 0x90, 0x5f, 0x36, 0x3b, 0x77, 0x00, 0xdb, 0xbf, 0x66, - 0x68, 0x16, 0x0e, 0xac, 0xf9, 0xca, 0x24, 0x53, 0xda, 0xde, 0xe2, 0xe8, 0x7e, 0x00, 0x72, 0x92, - 0xe0, 0x88, 0xe3, 0x03, 0x24, 0x9c, 0xc9, 0x71, 0xf5, 0x5e, 0x39, 0x6e, 0x41, 0xa7, 0x00, 0xad, - 0xb8, 0xb8, 0x14, 0xc8, 0x29, 0x86, 0xf8, 0xa0, 0x8a, 0x15, 0x8f, 0x46, 0x49, 0x59, 0x76, 0x49, - 0x59, 0x82, 0x41, 0xa1, 0x94, 0x66, 0x30, 0x83, 0xce, 0x31, 0x63, 0x74, 0x1a, 0xdd, 0xc4, 0x61, - 0x3a, 0xc3, 0x05, 0x85, 0x2e, 0xd4, 0xfd, 0x38, 0xd5, 0x2d, 0xaa, 0x7b, 0xea, 0x40, 0xf6, 0x01, - 0xfc, 0x38, 0x0c, 0xd1, 0xe7, 0x34, 0x8e, 0x34, 0x01, 0xc3, 0x42, 0xfa, 0xd0, 0x4a, 0x70, 0x1e, - 0x52, 0x7f, 0x24, 0x03, 0xd4, 0x6e, 0x98, 0x26, 0xf7, 0x33, 0x74, 0x8b, 0xe5, 0xf4, 0x50, 0x96, - 0x6a, 0x50, 0xac, 0x77, 0x12, 0xea, 0x5a, 0xe2, 0x53, 0xee, 0x66, 0x3a, 0x0e, 0xa9, 0x3f, 0x14, - 0x0e, 0x5b, 0xef, 0xa6, 0xb4, 0x5c, 0x27, 0x61, 0xce, 0xbc, 0x66, 0x30, 0x17, 0xa3, 0xbd, 0x9e, - 0x07, 0x7f, 0x6b, 0xb4, 0x05, 0x68, 0x75, 0xa3, 0xc1, 0x8f, 0x3a, 0xb4, 0xaf, 0x70, 0xf4, 0x05, - 0x31, 0x10, 0x5b, 0x98, 0x90, 0x29, 0x74, 0xab, 0x1e, 0x76, 0x72, 0x90, 0xe3, 0xde, 0xf3, 0x4b, - 0xd2, 0x7b, 0xf6, 0xbb, 0x30, 0x3d, 0xd0, 0x15, 0x72, 0x01, 0x2d, 0xe3, 0x19, 0x27, 0xbb, 0x46, - 0x62, 0xe9, 0x17, 0xa1, 0xb7, 0xb7, 0xc4, 0x9b, 0xa1, 0x8d, 0x80, 0x94, 0x25, 0x4d, 0x9e, 0xe4, - 0x69, 0x4b, 0x5f, 0x96, 0xde, 0xd3, 0xfb, 0x83, 0xb2, 0x12, 0xd7, 0xb0, 0x59, 0xd4, 0x2a, 0x79, - 0x54, 0xc8, 0x2c, 0xeb, 0xbe, 0xd7, 0x5f, 0x1e, 0x60, 0xf6, 0xc1, 0xd0, 0x9c, 0xd9, 0x87, 0xb2, - 0xca, 0xcd, 0x3e, 0x54, 0x09, 0x55, 0xa2, 0x19, 0x63, 0x36, 0xd1, 0xca, 0x8b, 0x65, 0xa2, 0x55, - 0xec, 0x86, 0x42, 0x33, 0xd4, 0x68, 0xa2, 0x95, 0xdf, 0x03, 0x13, 0xad, 0x4a, 0xc2, 0x2b, 0xe4, - 0x12, 0xda, 0xa6, 0xaa, 0x88, 0x91, 0x50, 0x21, 0xee, 0xde, 0xfe, 0x32, 0xf7, 0x02, 0x70, 0xdc, - 0x90, 0x7f, 0x73, 0x5e, 0xfd, 0x0c, 0x00, 0x00, 0xff, 0xff, 0x9c, 0xf1, 0x24, 0x43, 0xf5, 0x08, - 0x00, 0x00, + 0x10, 0xae, 0xe3, 0x24, 0x6d, 0x26, 0x69, 0xcf, 0xd1, 0x26, 0xed, 0xf1, 0x49, 0x7f, 0x08, 0x86, + 0xa2, 0x22, 0xa4, 0x0a, 0x85, 0x1b, 0x2e, 0xa9, 0xda, 0x52, 0x21, 0x15, 0x55, 0x72, 0x55, 0x24, + 0xae, 0xa2, 0xc4, 0x9e, 0x84, 0x55, 0x1d, 0x3b, 0x78, 0xd7, 0xa0, 0x72, 0x0b, 0xaf, 0xc2, 0x4b, + 0xf0, 0x74, 0x68, 0x7f, 0xe2, 0xac, 0xb1, 0xd3, 0x9f, 0x0b, 0xee, 0x76, 0x67, 0x76, 0xbe, 0xf9, + 0x76, 0x66, 0xbe, 0xb5, 0xa1, 0x39, 0xa6, 0x21, 0x26, 0x87, 0xb3, 0x24, 0xe6, 0x31, 0x59, 0x93, + 0x9b, 0xc1, 0x6c, 0xe4, 0x5e, 0xc0, 0xf6, 0x79, 0x1c, 0x5f, 0xa7, 0xb3, 0x13, 0x9a, 0xa0, 0xcf, + 0xe3, 0xe4, 0xe6, 0x34, 0xe2, 0xc9, 0x8d, 0x87, 0x9f, 0x53, 0x64, 0x9c, 0xec, 0x40, 0x23, 0x98, + 0x3b, 0x1c, 0xab, 0x67, 0x1d, 0x34, 0xbc, 0x85, 0x81, 0x10, 0xa8, 0x46, 0xc3, 0x29, 0x3a, 0x15, + 0xe9, 0x90, 0x6b, 0xf7, 0x14, 0x76, 0xca, 0x01, 0xd9, 0x2c, 0x8e, 0x18, 0x92, 0x7d, 0xa8, 0xa1, + 0x30, 0x48, 0xb4, 0x66, 0xff, 0x9f, 0xc3, 0x39, 0x95, 0x43, 0x75, 0x4e, 0x79, 0xdd, 0x3e, 0x90, + 0x73, 0xca, 0xb8, 0xb0, 0x51, 0x64, 0xf7, 0xa2, 0xe3, 0xbe, 0x81, 0x76, 0x2e, 0x46, 0x67, 0x7c, + 0x0e, 0xab, 0xa8, 0x4c, 0x8e, 0xd5, 0xb3, 0xcb, 0x72, 0xce, 0xfd, 0xee, 0x4f, 0x0b, 0x6a, 0xd2, + 0x94, 0x5d, 0xcd, 0x5a, 0x5c, 0x8d, 0x3c, 0x86, 0x16, 0x65, 0x83, 0x05, 0x01, 0x71, 0xed, 0x35, + 0xaf, 0x49, 0x59, 0x76, 0x55, 0xf2, 0x02, 0xea, 0xfe, 0xa7, 0x34, 0xba, 0x66, 0x8e, 0x2d, 0x53, + 0xb5, 0x17, 0xa9, 0xde, 0xd2, 0x10, 0x8f, 0x85, 0xcf, 0xd3, 0x47, 0xc8, 0x6b, 0x80, 0x21, 0xe7, + 0x09, 0x1d, 0xa5, 0x1c, 0x99, 0x53, 0x95, 0xf5, 0x70, 0x8c, 0x80, 0x94, 0xe1, 0x51, 0xe6, 0xf7, + 0x8c, 0xb3, 0xee, 0x18, 0x1a, 0x19, 0x1c, 0xf9, 0x0f, 0x56, 0x45, 0xcc, 0x80, 0x06, 0x9a, 0x6d, + 0x5d, 0x6c, 0xdf, 0x05, 0x64, 0x0b, 0xea, 0xf1, 0x78, 0xcc, 0x90, 0x4b, 0xa6, 0xb6, 0xa7, 0x77, + 0xe2, 0x6e, 0x8c, 0x7e, 0x43, 0xc7, 0xee, 0x59, 0x07, 0x55, 0x4f, 0xae, 0x49, 0x07, 0x6a, 0x53, + 0x4e, 0xa7, 0x28, 0x69, 0xd8, 0x9e, 0xda, 0xb8, 0x3f, 0x2c, 0xd8, 0xc8, 0xd3, 0x20, 0xdb, 0xd0, + 0x90, 0xd9, 0x24, 0x82, 0x25, 0x11, 0xe4, 0x34, 0x5d, 0xe6, 0x50, 0x2a, 0x06, 0x4a, 0x16, 0x32, + 0x8d, 0x03, 0x95, 0x74, 0x5d, 0x85, 0xbc, 0x8f, 0x03, 0x24, 0xff, 0x82, 0x9d, 0xd2, 0x40, 0xa6, + 0x5d, 0xf7, 0xc4, 0x52, 0x58, 0x26, 0x34, 0x70, 0x6a, 0xca, 0x32, 0xa1, 0x81, 0x3b, 0x81, 0xff, + 0xcf, 0x50, 0xf6, 0xf5, 0xc6, 0x28, 0x88, 0x9e, 0x89, 0xb2, 0x4e, 0xed, 0x02, 0xcc, 0x86, 0x09, + 0x46, 0x5c, 0x74, 0x4b, 0x8f, 0x67, 0x43, 0x59, 0x4e, 0x68, 0x62, 0x56, 0xcc, 0x36, 0x2b, 0xe6, + 0x7e, 0xb7, 0xa0, 0x5b, 0x96, 0x49, 0x4f, 0x52, 0xbe, 0x61, 0xd6, 0xfd, 0x1b, 0x66, 0xcc, 0x45, + 0xe5, 0xce, 0xb9, 0x70, 0x5f, 0xc2, 0xe6, 0x19, 0x72, 0x69, 0x8f, 0x23, 0x8e, 0x11, 0x9f, 0x5f, + 0x75, 0x59, 0xa7, 0xdd, 0x3e, 0x6c, 0xfd, 0x19, 0xa1, 0x29, 0x3b, 0xb0, 0xea, 0x2b, 0x93, 0x0c, + 0x69, 0x79, 0xf3, 0xad, 0xfb, 0x11, 0xc8, 0x71, 0x82, 0x43, 0x8e, 0x0f, 0x10, 0x7c, 0x26, 0xde, + 0xca, 0xad, 0xe2, 0xdd, 0x84, 0x76, 0x0e, 0x5a, 0x71, 0x71, 0x29, 0x90, 0x13, 0x0c, 0xf1, 0x41, + 0x19, 0x4b, 0x9e, 0x98, 0x82, 0x0e, 0xed, 0x82, 0x0e, 0x05, 0x83, 0x5c, 0x2a, 0xcd, 0x60, 0x0a, + 0xed, 0x23, 0xc6, 0xe8, 0x24, 0xfa, 0x10, 0x87, 0xe9, 0x14, 0xe7, 0x14, 0x3a, 0x50, 0xf3, 0xe3, + 0x54, 0x97, 0xa8, 0xe6, 0xa9, 0x0d, 0xd9, 0x03, 0xf0, 0xe3, 0x30, 0x44, 0x9f, 0xd3, 0x38, 0xd2, + 0x04, 0x0c, 0x0b, 0xe9, 0x41, 0x33, 0xc1, 0x59, 0x48, 0xfd, 0xa1, 0x3c, 0xa0, 0x26, 0xc9, 0x34, + 0xb9, 0x5f, 0xa0, 0x93, 0x4f, 0xa7, 0x9b, 0xb2, 0x54, 0xb1, 0x42, 0x0c, 0x49, 0xa8, 0x73, 0x89, + 0xa5, 0x9c, 0xe4, 0x74, 0x14, 0x52, 0x7f, 0x20, 0x1c, 0xb6, 0x9e, 0x64, 0x69, 0xb9, 0x4a, 0xc2, + 0x05, 0xf3, 0xaa, 0xc1, 0x5c, 0xb4, 0xf6, 0x6a, 0x16, 0xfc, 0xad, 0xd6, 0xe6, 0xa0, 0xd5, 0x8d, + 0xfa, 0xbf, 0x6a, 0xd0, 0xba, 0xc4, 0xe1, 0x57, 0xc4, 0x40, 0x4c, 0x61, 0x42, 0x26, 0xd0, 0x29, + 0xfb, 0x0c, 0x90, 0xfd, 0x05, 0xee, 0x2d, 0xdf, 0x9d, 0xee, 0xb3, 0xbb, 0x8e, 0xe9, 0x86, 0xae, + 0x90, 0x73, 0x68, 0x1a, 0x8f, 0x3e, 0xd9, 0x31, 0x02, 0x0b, 0xdf, 0x8f, 0xee, 0xee, 0x12, 0x6f, + 0x86, 0x36, 0x04, 0x52, 0xd4, 0x3f, 0x79, 0xb2, 0x08, 0x5b, 0xfa, 0x0e, 0x75, 0x9f, 0xde, 0x7e, + 0x28, 0x4b, 0x71, 0x05, 0x1b, 0x79, 0xad, 0x92, 0x47, 0xb9, 0xc8, 0xa2, 0xee, 0xbb, 0xbd, 0xe5, + 0x07, 0xcc, 0x3a, 0x18, 0x9a, 0x33, 0xeb, 0x50, 0x54, 0xb9, 0x59, 0x87, 0x32, 0xa1, 0x4a, 0x34, + 0xa3, 0xcd, 0x26, 0x5a, 0x71, 0xb0, 0x4c, 0xb4, 0x92, 0xd9, 0x50, 0x68, 0x86, 0x1a, 0x4d, 0xb4, + 0xe2, 0x7b, 0x60, 0xa2, 0x95, 0x49, 0x78, 0x85, 0x5c, 0x40, 0xcb, 0x54, 0x15, 0x31, 0x02, 0x4a, + 0xc4, 0xdd, 0xdd, 0x5b, 0xe6, 0x9e, 0x03, 0x8e, 0xea, 0xf2, 0xa7, 0xe8, 0xd5, 0xef, 0x00, 0x00, + 0x00, 0xff, 0xff, 0xec, 0x38, 0x88, 0xb2, 0x23, 0x09, 0x00, 0x00, } diff --git a/weed/server/filer_grpc_server.go b/weed/server/filer_grpc_server.go index 8d643ed2d..67d287201 100644 --- a/weed/server/filer_grpc_server.go +++ b/weed/server/filer_grpc_server.go @@ -71,18 +71,20 @@ func (fs *FilerServer) GetEntryAttributes(ctx context.Context, req *filer_pb.Get } if !found { attributes.FileSize = 0 - } else { - attributes.FileSize = filer2.TotalSize(entry.Chunks) - attributes.FileMode = uint32(entry.Mode) - attributes.Uid = entry.Uid - attributes.Gid = entry.Gid - attributes.Mtime = entry.Mtime.Unix() + return nil, fmt.Errorf("file %s not found", fullpath) } - glog.V(0).Infof("GetEntryAttributes %v: %+v", fullpath, attributes) + attributes.FileSize = filer2.TotalSize(entry.Chunks) + attributes.FileMode = uint32(entry.Mode) + attributes.Uid = entry.Uid + attributes.Gid = entry.Gid + attributes.Mtime = entry.Mtime.Unix() + + glog.V(0).Infof("GetEntryAttributes %v size %d chunks %d: %+v", fullpath, attributes.FileSize, len(entry.Chunks), attributes) return &filer_pb.GetEntryAttributesResponse{ Attributes: attributes, + Chunks: entry.Chunks, }, nil } @@ -112,6 +114,7 @@ func (fs *FilerServer) CreateEntry(ctx context.Context, req *filer_pb.CreateEntr Uid: req.Entry.Attributes.Uid, Gid: req.Entry.Attributes.Gid, }, + Chunks: req.Entry.Chunks, }) if err == nil { From 69b9d8c3c25f7ab3b8c7ad54aeeddcea50548852 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Tue, 22 May 2018 10:18:09 -0700 Subject: [PATCH 18/83] fix boltdb variable usage --- weed/storage/needle_map_boltdb.go | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/weed/storage/needle_map_boltdb.go b/weed/storage/needle_map_boltdb.go index 96c29cab6..cd15607bb 100644 --- a/weed/storage/needle_map_boltdb.go +++ b/weed/storage/needle_map_boltdb.go @@ -75,8 +75,8 @@ func generateBoltDbFile(dbFileName string, indexFile *os.File) error { } func (m *BoltDbNeedleMap) Get(key uint64) (element *needle.NeedleValue, ok bool) { + var offset, size uint32 bytes := make([]byte, 8) - var data []byte util.Uint64toBytes(bytes, key) err := m.db.View(func(tx *bolt.Tx) error { bucket := tx.Bucket(boltdbBucket) @@ -84,15 +84,22 @@ func (m *BoltDbNeedleMap) Get(key uint64) (element *needle.NeedleValue, ok bool) return fmt.Errorf("Bucket %q not found!", boltdbBucket) } - data = bucket.Get(bytes) + data := bucket.Get(bytes) + + if len(data) != 8 { + glog.V(0).Infof("wrong data length: %d", len(data)) + return fmt.Errorf("wrong data length: %d", len(data)) + } + + offset = util.BytesToUint32(data[0:4]) + size = util.BytesToUint32(data[4:8]) + return nil }) - if err != nil || len(data) != 8 { + if err != nil { return nil, false } - offset := util.BytesToUint32(data[0:4]) - size := util.BytesToUint32(data[4:8]) return &needle.NeedleValue{Key: needle.Key(key), Offset: offset, Size: size}, true } From 468514f5256671bc7649c516f67c9996ae9a273a Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Wed, 23 May 2018 02:36:06 -0700 Subject: [PATCH 19/83] reduce logs --- weed/filer2/filechunks.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/weed/filer2/filechunks.go b/weed/filer2/filechunks.go index 834e014d9..65197d471 100644 --- a/weed/filer2/filechunks.go +++ b/weed/filer2/filechunks.go @@ -54,7 +54,7 @@ func FindUnusedFileChunks(oldChunks, newChunks []*filer_pb.FileChunk) (unused [] func logPrintf(name string, visibles []*visibleInterval) { - // return + return log.Printf("%s len %d", name, len(visibles)) for _, v := range visibles { @@ -82,7 +82,7 @@ func nonOverlappingVisibleIntervals(chunks []*filer_pb.FileChunk) (visibles []*v var minStopInterval, upToDateInterval *visibleInterval watermarkStart := chunks[0].Offset for _, chunk := range chunks { - log.Printf("checking chunk: [%d,%d)", chunk.Offset, chunk.Offset+int64(chunk.Size)) + // log.Printf("checking chunk: [%d,%d)", chunk.Offset, chunk.Offset+int64(chunk.Size)) logPrintf("parallelIntervals", parallelIntervals) for len(parallelIntervals) > 0 && watermarkStart < chunk.Offset { logPrintf("parallelIntervals loop 1", parallelIntervals) From e97c60cc101ef472440babd39a396deb1a6b3ed9 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Wed, 23 May 2018 02:36:19 -0700 Subject: [PATCH 20/83] avoid overwriting variables --- weed/filer2/memdb/memdb_store.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/weed/filer2/memdb/memdb_store.go b/weed/filer2/memdb/memdb_store.go index ba26eb163..34757b00f 100644 --- a/weed/filer2/memdb/memdb_store.go +++ b/weed/filer2/memdb/memdb_store.go @@ -34,7 +34,7 @@ func (filer *MemDbStore) InsertEntry(entry *filer2.Entry) (err error) { } func (filer *MemDbStore) UpdateEntry(entry *filer2.Entry) (err error) { - found, entry, err := filer.FindEntry(entry.FullPath) + found, _, err := filer.FindEntry(entry.FullPath) if !found { return fmt.Errorf("No such file: %s", entry.FullPath) } From 1675243f29c5f37ca698b36b8bd4e74d5f1671f4 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Wed, 23 May 2018 02:38:11 -0700 Subject: [PATCH 21/83] maybe speed up a little when loading index --- weed/storage/needle_map_memory.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/weed/storage/needle_map_memory.go b/weed/storage/needle_map_memory.go index f34a57849..6da4bf82b 100644 --- a/weed/storage/needle_map_memory.go +++ b/weed/storage/needle_map_memory.go @@ -53,14 +53,14 @@ func doLoading(file *os.File, nm *NeedleMap) (*NeedleMap, error) { nm.FileCounter++ nm.FileByteCounter = nm.FileByteCounter + uint64(size) oldOffset, oldSize := nm.m.Set(needle.Key(key), offset, size) - glog.V(3).Infoln("reading key", key, "offset", offset*NeedlePaddingSize, "size", size, "oldSize", oldSize) + // glog.V(3).Infoln("reading key", key, "offset", offset*NeedlePaddingSize, "size", size, "oldSize", oldSize) if oldOffset > 0 && oldSize != TombstoneFileSize { nm.DeletionCounter++ nm.DeletionByteCounter = nm.DeletionByteCounter + uint64(oldSize) } } else { oldSize := nm.m.Delete(needle.Key(key)) - glog.V(3).Infoln("removing key", key, "offset", offset*NeedlePaddingSize, "size", size, "oldSize", oldSize) + // glog.V(3).Infoln("removing key", key, "offset", offset*NeedlePaddingSize, "size", size, "oldSize", oldSize) nm.DeletionCounter++ nm.DeletionByteCounter = nm.DeletionByteCounter + uint64(oldSize) } @@ -86,7 +86,7 @@ func WalkIndexFile(r *os.File, fn func(key uint64, offset, size uint32) error) e for count > 0 && e == nil || e == io.EOF { for i = 0; i+16 <= count; i += 16 { - key, offset, size = idxFileEntry(bytes[i : i+16]) + key, offset, size = idxFileEntry(bytes[i: i+16]) if e = fn(key, offset, size); e != nil { return e } From 536559f62dd937da7780e743cd7056e9682e2fb6 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Wed, 23 May 2018 03:08:46 -0700 Subject: [PATCH 22/83] copy works, edit somehow still fails --- weed/filesys/dir.go | 16 ++- weed/filesys/file.go | 206 ++++++------------------------- weed/filesys/filehandle.go | 176 ++++++++++++++++++++++++++ weed/server/filer_grpc_server.go | 34 +++-- 4 files changed, 252 insertions(+), 180 deletions(-) create mode 100644 weed/filesys/filehandle.go diff --git a/weed/filesys/dir.go b/weed/filesys/dir.go index 51b617689..8d07705c6 100644 --- a/weed/filesys/dir.go +++ b/weed/filesys/dir.go @@ -114,9 +114,19 @@ func (dir *Dir) Create(ctx context.Context, req *fuse.CreateRequest, }) if err == nil { - node := dir.newFile(req.Name, nil) - dir.NodeMap[req.Name] = node - return node, node, nil + file := dir.newFile(req.Name, nil) + dir.NodeMap[req.Name] = file + return file, &FileHandle{ + wfs: file.wfs, + dirPath: file.dir.Path, + name: file.Name, + RequestId: req.Header.ID, + NodeId: req.Header.Node, + Uid: req.Uid, + Gid: req.Gid, + attributes: file.attributes, + Chunks: file.Chunks, + }, nil } return nil, nil, err diff --git a/weed/filesys/file.go b/weed/filesys/file.go index 9604d6fb0..55edaf515 100644 --- a/weed/filesys/file.go +++ b/weed/filesys/file.go @@ -11,20 +11,12 @@ import ( "path/filepath" "os" "time" - "bytes" - "github.com/chrislusf/seaweedfs/weed/operation" "github.com/chrislusf/seaweedfs/weed/filer2" ) var _ = fs.Node(&File{}) var _ = fs.NodeOpener(&File{}) var _ = fs.NodeFsyncer(&File{}) -var _ = fs.Handle(&File{}) -var _ = fs.HandleReadAller(&File{}) -// var _ = fs.HandleReader(&File{}) -var _ = fs.HandleFlusher(&File{}) -var _ = fs.HandleWriter(&File{}) -var _ = fs.HandleReleaser(&File{}) var _ = fs.NodeSetattrer(&File{}) type File struct { @@ -32,44 +24,42 @@ type File struct { Name string dir *Dir wfs *WFS - isOpened bool attributes *filer_pb.FuseAttributes } func (file *File) Attr(context context.Context, attr *fuse.Attr) error { - if !file.isOpened { - fullPath := filepath.Join(file.dir.Path, file.Name) - item := file.wfs.listDirectoryEntriesCache.Get(fullPath) - if item != nil { - entry := item.Value().(*filer_pb.Entry) - file.Chunks = entry.Chunks - file.attributes = entry.Attributes - glog.V(1).Infof("read cached file %v attributes", file.Name) - } else { - err := file.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error { + fullPath := filepath.Join(file.dir.Path, file.Name) + item := file.wfs.listDirectoryEntriesCache.Get(fullPath) + if item != nil { + entry := item.Value().(*filer_pb.Entry) + file.Chunks = entry.Chunks + file.attributes = entry.Attributes + glog.V(1).Infof("file attr read cached %v attributes", file.Name) + } else { + err := file.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error { + + request := &filer_pb.GetEntryAttributesRequest{ + Name: file.Name, + ParentDir: file.dir.Path, + } - request := &filer_pb.GetEntryAttributesRequest{ - Name: file.Name, - ParentDir: file.dir.Path, - } + resp, err := client.GetEntryAttributes(context, request) + if err != nil { + glog.V(0).Infof("file attr read file %v: %v", request, err) + return err + } - glog.V(1).Infof("read file: %v", request) - resp, err := client.GetEntryAttributes(context, request) - if err != nil { - glog.V(0).Infof("read file %v: %v", request, err) - return err - } + file.attributes = resp.Attributes + file.Chunks = resp.Chunks - file.attributes = resp.Attributes - file.Chunks = resp.Chunks + glog.V(1).Infof("file attr %v %+v: %d", fullPath, file.attributes, filer2.TotalSize(file.Chunks)) - return nil - }) + return nil + }) - if err != nil { - return err - } + if err != nil { + return err } } @@ -84,19 +74,29 @@ func (file *File) Attr(context context.Context, attr *fuse.Attr) error { } func (file *File) Open(ctx context.Context, req *fuse.OpenRequest, resp *fuse.OpenResponse) (fs.Handle, error) { + fullPath := filepath.Join(file.dir.Path, file.Name) - fmt.Printf("Open %v %+v\n", fullPath, req) - file.isOpened = true + glog.V(3).Infof("file open %v %+v", fullPath, req) - return file, nil + return &FileHandle{ + wfs: file.wfs, + dirPath: file.dir.Path, + name: file.Name, + RequestId: req.Header.ID, + NodeId: req.Header.Node, + Uid: req.Uid, + Gid: req.Gid, + attributes: file.attributes, + Chunks: file.Chunks, + }, nil } func (file *File) Setattr(ctx context.Context, req *fuse.SetattrRequest, resp *fuse.SetattrResponse) error { fullPath := filepath.Join(file.dir.Path, file.Name) - fmt.Printf("Setattr %v %+v\n", fullPath, req) + glog.V(3).Infof("file setattr %v %+v", fullPath, req) if req.Valid.Size() { if req.Size == 0 { @@ -125,134 +125,10 @@ func (file *File) Setattr(ctx context.Context, req *fuse.SetattrRequest, resp *f } -func (file *File) ReadAll(ctx context.Context) (content []byte, err error) { - - fmt.Printf("read all file %+v/%v\n", file.dir.Path, file.Name) - - if len(file.Chunks) == 0 { - glog.V(0).Infof("empty file %v/%v", file.dir.Path, file.Name) - return - } - - err = file.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error { - - // FIXME: need to either use Read() or implement differently - chunks, _ := filer2.CompactFileChunks(file.Chunks) - glog.V(1).Infof("read file %v/%v %d/%d chunks", file.dir.Path, file.Name, len(chunks), len(file.Chunks)) - request := &filer_pb.GetFileContentRequest{ - FileId: chunks[0].FileId, - } - - glog.V(1).Infof("read file content %d chunk %s [%d,%d): %v", len(chunks), - chunks[0].FileId, chunks[0].Offset, chunks[0].Offset+int64(chunks[0].Size), request) - resp, err := client.GetFileContent(ctx, request) - if err != nil { - return err - } - - content = resp.Content - - return nil - }) - - return content, err -} - func (file *File) Fsync(ctx context.Context, req *fuse.FsyncRequest) error { // fsync works at OS level // write the file chunks to the filer - fmt.Printf("flush file %+v\n", req) - - return nil -} - -func (file *File) Flush(ctx context.Context, req *fuse.FlushRequest) error { - // fflush works at file level - // send the data to the OS - glog.V(3).Infof("file flush %v", req) - - if len(file.Chunks) == 0 { - glog.V(2).Infof("%x file %s/%s flush skipping empty: %v", file, file.dir.Path, file.Name, req) - return nil - } - - err := file.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error { - - request := &filer_pb.UpdateEntryRequest{ - Directory: file.dir.Path, - Entry: &filer_pb.Entry{ - Name: file.Name, - Attributes: file.attributes, - Chunks: file.Chunks, - }, - } - - glog.V(1).Infof("%s/%s set chunks: %v", file.dir.Path, file.Name, len(file.Chunks)) - if _, err := client.UpdateEntry(ctx, request); err != nil { - return fmt.Errorf("update file: %v", err) - } - - return nil - }) - - return err -} - -func (file *File) Write(ctx context.Context, req *fuse.WriteRequest, resp *fuse.WriteResponse) error { - // write the request to volume servers - // fmt.Printf("write file %+v\n", req) - - var fileId, host string - - if err := file.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error { - - request := &filer_pb.AssignVolumeRequest{ - Count: 1, - Replication: "000", - Collection: "", - } - - glog.V(1).Infof("assign volume: %v", request) - resp, err := client.AssignVolume(ctx, request) - if err != nil { - return err - } - - fileId, host = resp.FileId, resp.Url - - return nil - }); err != nil { - return fmt.Errorf("filer assign volume: %v", err) - } - - fileUrl := fmt.Sprintf("http://%s/%s", host, fileId) - bufReader := bytes.NewReader(req.Data) - uploadResult, err := operation.Upload(fileUrl, file.Name, bufReader, false, "application/octet-stream", nil, "") - if err != nil { - return fmt.Errorf("upload data: %v", err) - } - if uploadResult.Error != "" { - return fmt.Errorf("upload result: %v", uploadResult.Error) - } - - resp.Size = int(uploadResult.Size) - - file.Chunks = append(file.Chunks, &filer_pb.FileChunk{ - FileId: fileId, - Offset: req.Offset, - Size: uint64(uploadResult.Size), - Mtime: time.Now().UnixNano(), - }) - - glog.V(1).Infof("uploaded %s/%s to: %v, [%d,%d)", file.dir.Path, file.Name, fileUrl, req.Offset, req.Offset+int64(resp.Size)) - - return nil -} - -func (file *File) Release(ctx context.Context, req *fuse.ReleaseRequest) error { - - fmt.Printf("release file %+v\n", req) - file.isOpened = false + glog.V(3).Infof("fsync file %+v\n", req) return nil } diff --git a/weed/filesys/filehandle.go b/weed/filesys/filehandle.go new file mode 100644 index 000000000..5599f53a8 --- /dev/null +++ b/weed/filesys/filehandle.go @@ -0,0 +1,176 @@ +package filesys + +import ( + "bazil.org/fuse/fs" + "fmt" + "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" + "github.com/chrislusf/seaweedfs/weed/filer2" + "context" + "github.com/chrislusf/seaweedfs/weed/glog" + "bazil.org/fuse" + "bytes" + "github.com/chrislusf/seaweedfs/weed/operation" + "time" +) + +type FileHandle struct { + // cache file has been written to + dirty bool + + cachePath string + + handle uint64 + + wfs *WFS + dirPath string + name string + RequestId fuse.RequestID // unique ID for request + NodeId fuse.NodeID // file or directory the request is about + Uid uint32 // user ID of process making request + Gid uint32 // group ID of process making request + attributes *filer_pb.FuseAttributes + Chunks []*filer_pb.FileChunk +} + +var _ = fs.Handle(&FileHandle{}) +var _ = fs.HandleReadAller(&FileHandle{}) +// var _ = fs.HandleReader(&FileHandle{}) +var _ = fs.HandleFlusher(&FileHandle{}) +var _ = fs.HandleWriter(&FileHandle{}) +var _ = fs.HandleReleaser(&FileHandle{}) + +func (fh *FileHandle) ReadAll(ctx context.Context) (content []byte, err error) { + + glog.V(3).Infof("read all fh %+v/%v", fh.dirPath, fh.name) + + if len(fh.Chunks) == 0 { + glog.V(0).Infof("empty fh %v/%v", fh.dirPath, fh.name) + return + } + + err = fh.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error { + + // FIXME: need to either use Read() or implement differently + chunks, _ := filer2.CompactFileChunks(fh.Chunks) + glog.V(1).Infof("read fh %v/%v %d/%d chunks", fh.dirPath, fh.name, len(chunks), len(fh.Chunks)) + request := &filer_pb.GetFileContentRequest{ + FileId: chunks[0].FileId, + } + + glog.V(1).Infof("read fh content %d chunk %s [%d,%d): %v", len(chunks), + chunks[0].FileId, chunks[0].Offset, chunks[0].Offset+int64(chunks[0].Size), request) + resp, err := client.GetFileContent(ctx, request) + if err != nil { + return err + } + + content = resp.Content + + return nil + }) + + return content, err +} + +// Write to the file handle +func (fh *FileHandle) Write(ctx context.Context, req *fuse.WriteRequest, resp *fuse.WriteResponse) error { + // write the request to volume servers + // glog.V(3).Infof("write fh %+v", req) + + var fileId, host string + + if err := fh.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error { + + request := &filer_pb.AssignVolumeRequest{ + Count: 1, + Replication: "000", + Collection: "", + } + + glog.V(1).Infof("assign volume: %v", request) + resp, err := client.AssignVolume(ctx, request) + if err != nil { + return err + } + + fileId, host = resp.FileId, resp.Url + + return nil + }); err != nil { + return fmt.Errorf("filer assign volume: %v", err) + } + + fileUrl := fmt.Sprintf("http://%s/%s", host, fileId) + bufReader := bytes.NewReader(req.Data) + uploadResult, err := operation.Upload(fileUrl, fh.name, bufReader, false, "application/octet-stream", nil, "") + if err != nil { + return fmt.Errorf("upload data: %v", err) + } + if uploadResult.Error != "" { + return fmt.Errorf("upload result: %v", uploadResult.Error) + } + + resp.Size = int(uploadResult.Size) + + fh.Chunks = append(fh.Chunks, &filer_pb.FileChunk{ + FileId: fileId, + Offset: req.Offset, + Size: uint64(uploadResult.Size), + Mtime: time.Now().UnixNano(), + }) + + glog.V(1).Infof("uploaded %s/%s to: %v, [%d,%d)", fh.dirPath, fh.name, fileUrl, req.Offset, req.Offset+int64(resp.Size)) + + fh.dirty = true + + return nil +} + +func (fh *FileHandle) Release(ctx context.Context, req *fuse.ReleaseRequest) error { + + glog.V(3).Infof("release fh %+v/%v", fh.dirPath, fh.name) + + return nil +} + +// Flush - experimenting with uploading at flush, this slows operations down till it has been +// completely flushed +func (fh *FileHandle) Flush(ctx context.Context, req *fuse.FlushRequest) error { + // fflush works at fh level + // send the data to the OS + glog.V(3).Infof("fh flush %v", req) + + if !fh.dirty { + return nil + } + + if len(fh.Chunks) == 0 { + glog.V(2).Infof("fh %s/%s flush skipping empty: %v", fh.dirPath, fh.name, req) + return nil + } + + err := fh.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error { + + request := &filer_pb.UpdateEntryRequest{ + Directory: fh.dirPath, + Entry: &filer_pb.Entry{ + Name: fh.name, + Attributes: fh.attributes, + Chunks: fh.Chunks, + }, + } + + glog.V(1).Infof("%s/%s set chunks: %v", fh.dirPath, fh.name, len(fh.Chunks)) + if _, err := client.UpdateEntry(ctx, request); err != nil { + return fmt.Errorf("update fh: %v", err) + } + + return nil + }) + + if err == nil { + fh.dirty = false + } + + return err +} diff --git a/weed/server/filer_grpc_server.go b/weed/server/filer_grpc_server.go index 67d287201..da02ce169 100644 --- a/weed/server/filer_grpc_server.go +++ b/weed/server/filer_grpc_server.go @@ -80,7 +80,7 @@ func (fs *FilerServer) GetEntryAttributes(ctx context.Context, req *filer_pb.Get attributes.Gid = entry.Gid attributes.Mtime = entry.Mtime.Unix() - glog.V(0).Infof("GetEntryAttributes %v size %d chunks %d: %+v", fullpath, attributes.FileSize, len(entry.Chunks), attributes) + glog.V(3).Infof("GetEntryAttributes %v size %d chunks %d: %+v", fullpath, attributes.FileSize, len(entry.Chunks), attributes) return &filer_pb.GetEntryAttributesResponse{ Attributes: attributes, @@ -139,19 +139,29 @@ func (fs *FilerServer) UpdateEntry(ctx context.Context, req *filer_pb.UpdateEntr chunks, garbages := filer2.CompactFileChunks(req.Entry.Chunks) - err = fs.filer.UpdateEntry(&filer2.Entry{ + newEntry := &filer2.Entry{ FullPath: filer2.FullPath(filepath.Join(req.Directory, req.Entry.Name)), - Attr: filer2.Attr{ - Mtime: time.Unix(req.Entry.Attributes.Mtime, 0), - Crtime: time.Unix(req.Entry.Attributes.Mtime, 0), - Mode: os.FileMode(req.Entry.Attributes.FileMode), - Uid: req.Entry.Attributes.Uid, - Gid: req.Entry.Attributes.Gid, - }, - Chunks: chunks, - }) + Attr: entry.Attr, + Chunks: chunks, + } - if err == nil { + glog.V(3).Infof("updating %s: %+v, chunks %d: %v => %+v, chunks %d: %v", + fullpath, entry.Attr, len(entry.Chunks), entry.Chunks, + req.Entry.Attributes, len(req.Entry.Chunks), req.Entry.Chunks) + + if req.Entry.Attributes != nil { + if req.Entry.Attributes.Mtime != 0 { + newEntry.Attr.Mtime = time.Unix(req.Entry.Attributes.Mtime, 0) + } + if req.Entry.Attributes.FileMode != 0 { + newEntry.Attr.Mode = os.FileMode(req.Entry.Attributes.FileMode) + } + newEntry.Attr.Uid = req.Entry.Attributes.Uid + newEntry.Attr.Gid = req.Entry.Attributes.Gid + + } + + if err = fs.filer.UpdateEntry(newEntry); err == nil { for _, garbage := range unusedChunks { glog.V(0).Infof("deleting %s old chunk: %v, [%d, %d)", fullpath, garbage.FileId, garbage.Offset, garbage.Offset+int64(garbage.Size)) operation.DeleteFile(fs.master, garbage.FileId, fs.jwt(garbage.FileId)) From d84b80b79510359fb1404bd13f91ad64173e8a8b Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Wed, 23 May 2018 03:23:47 -0700 Subject: [PATCH 23/83] similar changes as master --- weed/compress/compression_test.go | 45 ---------------------------- weed/compress/delta_binary_pack32.go | 32 -------------------- weed/glide.lock | 6 ---- weed/glide.yaml | 4 --- 4 files changed, 87 deletions(-) delete mode 100644 weed/compress/compression_test.go delete mode 100644 weed/compress/delta_binary_pack32.go diff --git a/weed/compress/compression_test.go b/weed/compress/compression_test.go deleted file mode 100644 index 83b7c0055..000000000 --- a/weed/compress/compression_test.go +++ /dev/null @@ -1,45 +0,0 @@ -package compress - -import ( - "math/rand" - "testing" -) - -func TestSortedData(t *testing.T) { - data := make([]int32, 102400) - for i := 1; i < len(data); i++ { - data[i] = data[i-1] + rand.Int31n(15) - } - testCompressAndUncompress(t, data, "Sorted data") -} - -func TestUnsortedData(t *testing.T) { - data := make([]int32, 102400) - for i := 0; i < len(data); i++ { - data[i] = rand.Int31n(255) - } - testCompressAndUncompress(t, data, "Unsorted data") -} - -func testCompressAndUncompress(t *testing.T, data []int32, desc string) { - - compressed_data, err := Compress32(data) - if err != nil { - t.Fatal("Compress error", err.Error()) - } - uncompressed_data, err := Uncompress32(compressed_data, make([]int32, len(data)*2)) - if err != nil { - t.Fatal("Compress error", err.Error()) - } - if len(uncompressed_data) != len(data) { - t.Fatal("Len differs", len(data), len(uncompressed_data)) - } - for i := 0; i < len(data); i++ { - if data[i] != uncompressed_data[i] { - t.Fatal("Data differs:", i, data[i], uncompressed_data[i]) - } - } - - println(desc, " Data length:", len(data), " => Compressed length:", len(compressed_data)) - -} diff --git a/weed/compress/delta_binary_pack32.go b/weed/compress/delta_binary_pack32.go deleted file mode 100644 index 1da0e427a..000000000 --- a/weed/compress/delta_binary_pack32.go +++ /dev/null @@ -1,32 +0,0 @@ -package compress - -import ( - "github.com/dataence/encoding/cursor" - "github.com/dataence/encoding/delta/bp32" -) - -// Compress compresses in[]int32 to out[]int32 -func Compress32(in []int32) (out []int32, err error) { - out = make([]int32, len(in)*2) - inpos := cursor.New() - outpos := cursor.New() - - if err = bp32.New().Compress(in, inpos, len(in), out, outpos); err != nil { - return nil, err - } - - return out[:outpos.Get()], nil -} - -// Uncompress uncompresses in[]int32 to out[]int32 -func Uncompress32(in []int32, buffer []int32) (out []int32, err error) { - out = buffer - inpos := cursor.New() - outpos := cursor.New() - - if err = bp32.New().Uncompress(in, inpos, len(in), out, outpos); err != nil { - return nil, err - } - - return out[:outpos.Get()], nil -} diff --git a/weed/glide.lock b/weed/glide.lock index 52bdedc54..ad0e1bf8e 100644 --- a/weed/glide.lock +++ b/weed/glide.lock @@ -12,12 +12,6 @@ imports: version: 5f7ddd8f479583daf05879d3d3b174aa202c8fb7 subpackages: - protobuf -- name: github.com/dataence/encoding - version: b90e310a0325f9b765b4be7220df3642ad93ad8d - subpackages: - - bitpacking - - cursor - - delta/bp32 - name: github.com/dgrijalva/jwt-go version: 06ea1031745cb8b3dab3f6a236daf2b0aa468b7e - name: github.com/disintegration/imaging diff --git a/weed/glide.yaml b/weed/glide.yaml index 36737fb21..7903a6728 100644 --- a/weed/glide.yaml +++ b/weed/glide.yaml @@ -6,10 +6,6 @@ import: - package: github.com/boltdb/bolt version: ^1.3.1 - package: github.com/chrislusf/raft -- package: github.com/dataence/encoding - subpackages: - - cursor - - delta/bp32 - package: github.com/dgrijalva/jwt-go version: ^3.2.0 - package: github.com/disintegration/imaging From 849b6ec28d96540d530cf18c9f6fe3a08c1f5755 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Wed, 23 May 2018 20:55:24 -0700 Subject: [PATCH 24/83] seems editing already working Need to handle multiple chunks read. Need to cache local file changes. --- weed/filesys/file.go | 6 +++--- weed/filesys/filehandle.go | 13 +++++++++---- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/weed/filesys/file.go b/weed/filesys/file.go index 55edaf515..d7f0d6bc0 100644 --- a/weed/filesys/file.go +++ b/weed/filesys/file.go @@ -77,7 +77,7 @@ func (file *File) Open(ctx context.Context, req *fuse.OpenRequest, resp *fuse.Op fullPath := filepath.Join(file.dir.Path, file.Name) - glog.V(3).Infof("file open %v %+v", fullPath, req) + glog.V(3).Infof("%v file open %+v", fullPath, req) return &FileHandle{ wfs: file.wfs, @@ -96,7 +96,7 @@ func (file *File) Open(ctx context.Context, req *fuse.OpenRequest, resp *fuse.Op func (file *File) Setattr(ctx context.Context, req *fuse.SetattrRequest, resp *fuse.SetattrResponse) error { fullPath := filepath.Join(file.dir.Path, file.Name) - glog.V(3).Infof("file setattr %v %+v", fullPath, req) + glog.V(3).Infof("%v file setattr %+v", fullPath, req) if req.Valid.Size() { if req.Size == 0 { @@ -128,7 +128,7 @@ func (file *File) Setattr(ctx context.Context, req *fuse.SetattrRequest, resp *f func (file *File) Fsync(ctx context.Context, req *fuse.FsyncRequest) error { // fsync works at OS level // write the file chunks to the filer - glog.V(3).Infof("fsync file %+v\n", req) + glog.V(3).Infof("%s/%s fsync file %+v", file.dir.Path, file.Name, req) return nil } diff --git a/weed/filesys/filehandle.go b/weed/filesys/filehandle.go index 5599f53a8..55d574342 100644 --- a/weed/filesys/filehandle.go +++ b/weed/filesys/filehandle.go @@ -41,7 +41,7 @@ var _ = fs.HandleReleaser(&FileHandle{}) func (fh *FileHandle) ReadAll(ctx context.Context) (content []byte, err error) { - glog.V(3).Infof("read all fh %+v/%v", fh.dirPath, fh.name) + glog.V(3).Infof("%v/%v read all fh ", fh.dirPath, fh.name) if len(fh.Chunks) == 0 { glog.V(0).Infof("empty fh %v/%v", fh.dirPath, fh.name) @@ -53,6 +53,9 @@ func (fh *FileHandle) ReadAll(ctx context.Context) (content []byte, err error) { // FIXME: need to either use Read() or implement differently chunks, _ := filer2.CompactFileChunks(fh.Chunks) glog.V(1).Infof("read fh %v/%v %d/%d chunks", fh.dirPath, fh.name, len(chunks), len(fh.Chunks)) + for i, chunk := range chunks { + glog.V(1).Infof("read fh %v/%v %d/%d chunk %s [%d,%d)", fh.dirPath, fh.name, i, len(chunks), chunk.FileId, chunk.Offset, chunk.Offset+int64(chunk.Size)) + } request := &filer_pb.GetFileContentRequest{ FileId: chunks[0].FileId, } @@ -74,8 +77,10 @@ func (fh *FileHandle) ReadAll(ctx context.Context) (content []byte, err error) { // Write to the file handle func (fh *FileHandle) Write(ctx context.Context, req *fuse.WriteRequest, resp *fuse.WriteResponse) error { + // write the request to volume servers - // glog.V(3).Infof("write fh %+v", req) + + glog.V(3).Infof("%+v/%v write fh: %+v", fh.dirPath, fh.name, req) var fileId, host string @@ -128,7 +133,7 @@ func (fh *FileHandle) Write(ctx context.Context, req *fuse.WriteRequest, resp *f func (fh *FileHandle) Release(ctx context.Context, req *fuse.ReleaseRequest) error { - glog.V(3).Infof("release fh %+v/%v", fh.dirPath, fh.name) + glog.V(3).Infof("%+v/%v release fh", fh.dirPath, fh.name) return nil } @@ -138,7 +143,7 @@ func (fh *FileHandle) Release(ctx context.Context, req *fuse.ReleaseRequest) err func (fh *FileHandle) Flush(ctx context.Context, req *fuse.FlushRequest) error { // fflush works at fh level // send the data to the OS - glog.V(3).Infof("fh flush %v", req) + glog.V(3).Infof("%s/%s fh flush %v", fh.dirPath, fh.name, req) if !fh.dirty { return nil From 00d0274fd7c829f5d26c051f5832e0f602929b08 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Wed, 23 May 2018 22:28:54 -0700 Subject: [PATCH 25/83] prepare to read from multiple file chunks --- weed/filer2/filechunks.go | 21 +++++ weed/filer2/filechunks_test.go | 140 ++++++++++++++++++++++++++++++++- 2 files changed, 157 insertions(+), 4 deletions(-) diff --git a/weed/filer2/filechunks.go b/weed/filer2/filechunks.go index 65197d471..93cee81de 100644 --- a/weed/filer2/filechunks.go +++ b/weed/filer2/filechunks.go @@ -52,6 +52,27 @@ func FindUnusedFileChunks(oldChunks, newChunks []*filer_pb.FileChunk) (unused [] return } +func ReadFromChunks(chunks []*filer_pb.FileChunk, offset int64, size int) (views []*filer_pb.FileChunk) { + + visibles := nonOverlappingVisibleIntervals(chunks) + + stop := offset + int64(size) + + for _, chunk := range visibles { + if chunk.start <= offset && offset < chunk.stop { + views = append(views, &filer_pb.FileChunk{ + FileId: chunk.fileId, + Offset: offset - chunk.start, // offset is the data starting location in this file id + Size: uint64(min(chunk.stop, stop) - offset), + }) + offset = min(chunk.stop, stop) + } + } + + return views + +} + func logPrintf(name string, visibles []*visibleInterval) { return diff --git a/weed/filer2/filechunks_test.go b/weed/filer2/filechunks_test.go index b87b61d3b..9e39477be 100644 --- a/weed/filer2/filechunks_test.go +++ b/weed/filer2/filechunks_test.go @@ -9,10 +9,10 @@ import ( func TestCompactFileChunks(t *testing.T) { chunks := []*filer_pb.FileChunk{ - {Offset:10, Size:100, FileId:"abc", Mtime:50}, - {Offset:100, Size:100, FileId:"def", Mtime:100}, - {Offset:200, Size:100, FileId:"ghi", Mtime:200}, - {Offset:110, Size:200, FileId:"jkl", Mtime:300}, + {Offset: 10, Size: 100, FileId: "abc", Mtime: 50}, + {Offset: 100, Size: 100, FileId: "def", Mtime: 100}, + {Offset: 200, Size: 100, FileId: "ghi", Mtime: 200}, + {Offset: 110, Size: 200, FileId: "jkl", Mtime: 300}, } compacted, garbarge := CompactFileChunks(chunks) @@ -144,3 +144,135 @@ func TestIntervalMerging(t *testing.T) { } } + +func TestChunksReading(t *testing.T) { + + testcases := []struct { + Chunks []*filer_pb.FileChunk + Offset int64 + Size int + Expected []*filer_pb.FileChunk + }{ + // case 0: normal + { + Chunks: []*filer_pb.FileChunk{ + {Offset: 0, Size: 100, FileId: "abc", Mtime: 123}, + {Offset: 100, Size: 100, FileId: "asdf", Mtime: 134}, + {Offset: 200, Size: 100, FileId: "fsad", Mtime: 353}, + }, + Offset: 0, + Size: 250, + Expected: []*filer_pb.FileChunk{ + {Offset: 0, Size: 100, FileId: "abc"}, + {Offset: 0, Size: 100, FileId: "asdf"}, + {Offset: 0, Size: 50, FileId: "fsad"}, + }, + }, + // case 1: updates overwrite full chunks + { + Chunks: []*filer_pb.FileChunk{ + {Offset: 0, Size: 100, FileId: "abc", Mtime: 123}, + {Offset: 0, Size: 200, FileId: "asdf", Mtime: 134}, + }, + Offset: 50, + Size: 100, + Expected: []*filer_pb.FileChunk{ + {Offset: 50, Size: 100, FileId: "asdf"}, + }, + }, + // case 2: updates overwrite part of previous chunks + { + Chunks: []*filer_pb.FileChunk{ + {Offset: 0, Size: 100, FileId: "abc", Mtime: 123}, + {Offset: 0, Size: 50, FileId: "asdf", Mtime: 134}, + }, + Offset: 25, + Size: 50, + Expected: []*filer_pb.FileChunk{ + {Offset: 25, Size: 25, FileId: "asdf"}, + {Offset: 0, Size: 25, FileId: "abc"}, + }, + }, + // case 3: updates overwrite full chunks + { + Chunks: []*filer_pb.FileChunk{ + {Offset: 0, Size: 100, FileId: "abc", Mtime: 123}, + {Offset: 0, Size: 200, FileId: "asdf", Mtime: 134}, + {Offset: 50, Size: 250, FileId: "xxxx", Mtime: 154}, + }, + Offset: 0, + Size: 200, + Expected: []*filer_pb.FileChunk{ + {Offset: 0, Size: 50, FileId: "asdf"}, + {Offset: 0, Size: 150, FileId: "xxxx"}, + }, + }, + // case 4: updates far away from prev chunks + { + Chunks: []*filer_pb.FileChunk{ + {Offset: 0, Size: 100, FileId: "abc", Mtime: 123}, + {Offset: 0, Size: 200, FileId: "asdf", Mtime: 134}, + {Offset: 250, Size: 250, FileId: "xxxx", Mtime: 154}, + }, + Offset: 0, + Size: 400, + Expected: []*filer_pb.FileChunk{ + {Offset: 0, Size: 200, FileId: "asdf"}, + // {Offset: 0, Size: 150, FileId: "xxxx"}, // missing intervals should not happen + }, + }, + // case 5: updates overwrite full chunks + { + Chunks: []*filer_pb.FileChunk{ + {Offset: 0, Size: 100, FileId: "abc", Mtime: 123}, + {Offset: 0, Size: 200, FileId: "asdf", Mtime: 184}, + {Offset: 70, Size: 150, FileId: "abc", Mtime: 143}, + {Offset: 80, Size: 100, FileId: "xxxx", Mtime: 134}, + }, + Offset: 0, + Size: 220, + Expected: []*filer_pb.FileChunk{ + {Offset: 0, Size: 200, FileId: "asdf"}, + {Offset: 0, Size: 20, FileId: "abc"}, + }, + }, + // case 6: same updates + { + Chunks: []*filer_pb.FileChunk{ + {Offset: 0, Size: 100, FileId: "abc", Mtime: 123}, + {Offset: 0, Size: 100, FileId: "abc", Mtime: 123}, + {Offset: 0, Size: 100, FileId: "abc", Mtime: 123}, + }, + Offset: 0, + Size: 100, + Expected: []*filer_pb.FileChunk{ + {Offset: 0, Size: 100, FileId: "abc"}, + }, + }, + } + + for i, testcase := range testcases { + log.Printf("++++++++++ read test case %d ++++++++++++++++++++", i) + chunks := ReadFromChunks(testcase.Chunks, testcase.Offset, testcase.Size) + for x, chunk := range chunks { + log.Printf("read case %d, chunk %d, offset=%d, size=%d, fileId=%s", + i, x, chunk.Offset, chunk.Size, chunk.FileId) + if chunk.Offset != testcase.Expected[x].Offset { + t.Fatalf("failed on read case %d, chunk %d, Offset %d, expect %d", + i, x, chunk.Offset, testcase.Expected[x].Offset) + } + if chunk.Size != testcase.Expected[x].Size { + t.Fatalf("failed on read case %d, chunk %d, Size %d, expect %d", + i, x, chunk.Size, testcase.Expected[x].Size) + } + if chunk.FileId != testcase.Expected[x].FileId { + t.Fatalf("failed on read case %d, chunk %d, FileId %s, expect %s", + i, x, chunk.FileId, testcase.Expected[x].FileId) + } + } + if len(chunks) != len(testcase.Expected) { + t.Fatalf("failed to read test case %d, len %d expected %d", i, len(chunks), len(testcase.Expected)) + } + } + +} From d773e11c7a65903b3ee1adea801a20f91cb0c7aa Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Thu, 24 May 2018 01:22:37 -0700 Subject: [PATCH 26/83] file handler directly read from volume servers this mostly works fine now! next: need to cache files to local disk --- weed/filer2/filechunks.go | 18 +- weed/filer2/filechunks_test.go | 44 +++-- weed/filesys/filehandle.go | 99 ++++++++-- weed/pb/filer.proto | 30 ++- weed/pb/filer_pb/filer.pb.go | 302 ++++++++++++++++++++----------- weed/server/filer_grpc_server.go | 28 ++- weed/util/http_util.go | 30 +++ 7 files changed, 383 insertions(+), 168 deletions(-) diff --git a/weed/filer2/filechunks.go b/weed/filer2/filechunks.go index 93cee81de..6bdfbd48e 100644 --- a/weed/filer2/filechunks.go +++ b/weed/filer2/filechunks.go @@ -52,7 +52,14 @@ func FindUnusedFileChunks(oldChunks, newChunks []*filer_pb.FileChunk) (unused [] return } -func ReadFromChunks(chunks []*filer_pb.FileChunk, offset int64, size int) (views []*filer_pb.FileChunk) { +type ChunkView struct { + FileId string + Offset int64 + Size uint64 + LogicOffset int64 +} + +func ReadFromChunks(chunks []*filer_pb.FileChunk, offset int64, size int) (views []*ChunkView) { visibles := nonOverlappingVisibleIntervals(chunks) @@ -60,10 +67,11 @@ func ReadFromChunks(chunks []*filer_pb.FileChunk, offset int64, size int) (views for _, chunk := range visibles { if chunk.start <= offset && offset < chunk.stop { - views = append(views, &filer_pb.FileChunk{ - FileId: chunk.fileId, - Offset: offset - chunk.start, // offset is the data starting location in this file id - Size: uint64(min(chunk.stop, stop) - offset), + views = append(views, &ChunkView{ + FileId: chunk.fileId, + Offset: offset - chunk.start, // offset is the data starting location in this file id + Size: uint64(min(chunk.stop, stop) - offset), + LogicOffset: offset, }) offset = min(chunk.stop, stop) } diff --git a/weed/filer2/filechunks_test.go b/weed/filer2/filechunks_test.go index 9e39477be..24897215e 100644 --- a/weed/filer2/filechunks_test.go +++ b/weed/filer2/filechunks_test.go @@ -151,7 +151,7 @@ func TestChunksReading(t *testing.T) { Chunks []*filer_pb.FileChunk Offset int64 Size int - Expected []*filer_pb.FileChunk + Expected []*ChunkView }{ // case 0: normal { @@ -162,10 +162,10 @@ func TestChunksReading(t *testing.T) { }, Offset: 0, Size: 250, - Expected: []*filer_pb.FileChunk{ - {Offset: 0, Size: 100, FileId: "abc"}, - {Offset: 0, Size: 100, FileId: "asdf"}, - {Offset: 0, Size: 50, FileId: "fsad"}, + Expected: []*ChunkView{ + {Offset: 0, Size: 100, FileId: "abc", LogicOffset:0}, + {Offset: 0, Size: 100, FileId: "asdf", LogicOffset:100}, + {Offset: 0, Size: 50, FileId: "fsad", LogicOffset:200}, }, }, // case 1: updates overwrite full chunks @@ -176,8 +176,8 @@ func TestChunksReading(t *testing.T) { }, Offset: 50, Size: 100, - Expected: []*filer_pb.FileChunk{ - {Offset: 50, Size: 100, FileId: "asdf"}, + Expected: []*ChunkView{ + {Offset: 50, Size: 100, FileId: "asdf", LogicOffset:50}, }, }, // case 2: updates overwrite part of previous chunks @@ -188,9 +188,9 @@ func TestChunksReading(t *testing.T) { }, Offset: 25, Size: 50, - Expected: []*filer_pb.FileChunk{ - {Offset: 25, Size: 25, FileId: "asdf"}, - {Offset: 0, Size: 25, FileId: "abc"}, + Expected: []*ChunkView{ + {Offset: 25, Size: 25, FileId: "asdf", LogicOffset:25}, + {Offset: 0, Size: 25, FileId: "abc", LogicOffset:50}, }, }, // case 3: updates overwrite full chunks @@ -202,9 +202,9 @@ func TestChunksReading(t *testing.T) { }, Offset: 0, Size: 200, - Expected: []*filer_pb.FileChunk{ - {Offset: 0, Size: 50, FileId: "asdf"}, - {Offset: 0, Size: 150, FileId: "xxxx"}, + Expected: []*ChunkView{ + {Offset: 0, Size: 50, FileId: "asdf", LogicOffset:0}, + {Offset: 0, Size: 150, FileId: "xxxx", LogicOffset:50}, }, }, // case 4: updates far away from prev chunks @@ -216,8 +216,8 @@ func TestChunksReading(t *testing.T) { }, Offset: 0, Size: 400, - Expected: []*filer_pb.FileChunk{ - {Offset: 0, Size: 200, FileId: "asdf"}, + Expected: []*ChunkView{ + {Offset: 0, Size: 200, FileId: "asdf", LogicOffset:0}, // {Offset: 0, Size: 150, FileId: "xxxx"}, // missing intervals should not happen }, }, @@ -231,9 +231,9 @@ func TestChunksReading(t *testing.T) { }, Offset: 0, Size: 220, - Expected: []*filer_pb.FileChunk{ - {Offset: 0, Size: 200, FileId: "asdf"}, - {Offset: 0, Size: 20, FileId: "abc"}, + Expected: []*ChunkView{ + {Offset: 0, Size: 200, FileId: "asdf", LogicOffset:0}, + {Offset: 0, Size: 20, FileId: "abc", LogicOffset:200}, }, }, // case 6: same updates @@ -245,8 +245,8 @@ func TestChunksReading(t *testing.T) { }, Offset: 0, Size: 100, - Expected: []*filer_pb.FileChunk{ - {Offset: 0, Size: 100, FileId: "abc"}, + Expected: []*ChunkView{ + {Offset: 0, Size: 100, FileId: "abc", LogicOffset:0}, }, }, } @@ -269,6 +269,10 @@ func TestChunksReading(t *testing.T) { t.Fatalf("failed on read case %d, chunk %d, FileId %s, expect %s", i, x, chunk.FileId, testcase.Expected[x].FileId) } + if chunk.LogicOffset != testcase.Expected[x].LogicOffset { + t.Fatalf("failed on read case %d, chunk %d, LogicOffset %d, expect %d", + i, x, chunk.LogicOffset, testcase.Expected[x].LogicOffset) + } } if len(chunks) != len(testcase.Expected) { t.Fatalf("failed to read test case %d, len %d expected %d", i, len(chunks), len(testcase.Expected)) diff --git a/weed/filesys/filehandle.go b/weed/filesys/filehandle.go index 55d574342..c71f1ee36 100644 --- a/weed/filesys/filehandle.go +++ b/weed/filesys/filehandle.go @@ -11,6 +11,9 @@ import ( "bytes" "github.com/chrislusf/seaweedfs/weed/operation" "time" + "strings" + "sync" + "github.com/chrislusf/seaweedfs/weed/util" ) type FileHandle struct { @@ -33,46 +36,94 @@ type FileHandle struct { } var _ = fs.Handle(&FileHandle{}) -var _ = fs.HandleReadAller(&FileHandle{}) -// var _ = fs.HandleReader(&FileHandle{}) +// var _ = fs.HandleReadAller(&FileHandle{}) +var _ = fs.HandleReader(&FileHandle{}) var _ = fs.HandleFlusher(&FileHandle{}) var _ = fs.HandleWriter(&FileHandle{}) var _ = fs.HandleReleaser(&FileHandle{}) -func (fh *FileHandle) ReadAll(ctx context.Context) (content []byte, err error) { +func (fh *FileHandle) Read(ctx context.Context, req *fuse.ReadRequest, resp *fuse.ReadResponse) error { - glog.V(3).Infof("%v/%v read all fh ", fh.dirPath, fh.name) + glog.V(3).Infof("%v/%v read fh: [%d,%d)", fh.dirPath, fh.name, req.Offset, req.Offset+int64(req.Size)) if len(fh.Chunks) == 0 { glog.V(0).Infof("empty fh %v/%v", fh.dirPath, fh.name) - return + return fmt.Errorf("empty file %v/%v", fh.dirPath, fh.name) } - err = fh.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error { + buff := make([]byte, req.Size) - // FIXME: need to either use Read() or implement differently - chunks, _ := filer2.CompactFileChunks(fh.Chunks) - glog.V(1).Infof("read fh %v/%v %d/%d chunks", fh.dirPath, fh.name, len(chunks), len(fh.Chunks)) - for i, chunk := range chunks { - glog.V(1).Infof("read fh %v/%v %d/%d chunk %s [%d,%d)", fh.dirPath, fh.name, i, len(chunks), chunk.FileId, chunk.Offset, chunk.Offset+int64(chunk.Size)) - } - request := &filer_pb.GetFileContentRequest{ - FileId: chunks[0].FileId, - } + chunkViews := filer2.ReadFromChunks(fh.Chunks, req.Offset, req.Size) + + var vids []string + for _, chunkView := range chunkViews { + vids = append(vids, volumeId(chunkView.FileId)) + } + + vid2Locations := make(map[string]*filer_pb.Locations) + + err := fh.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error { - glog.V(1).Infof("read fh content %d chunk %s [%d,%d): %v", len(chunks), - chunks[0].FileId, chunks[0].Offset, chunks[0].Offset+int64(chunks[0].Size), request) - resp, err := client.GetFileContent(ctx, request) + glog.V(4).Infof("read fh lookup volume id locations: %v", vids) + resp, err := client.LookupVolume(ctx, &filer_pb.LookupVolumeRequest{ + VolumeIds: vids, + }) if err != nil { return err } - content = resp.Content + vid2Locations = resp.LocationsMap return nil }) - return content, err + if err != nil { + glog.V(3).Infof("%v/%v read fh lookup volume ids: %v", fh.dirPath, fh.name, err) + return fmt.Errorf("failed to lookup volume ids %v: %v", vids, err) + } + + var totalRead int64 + var wg sync.WaitGroup + for _, chunkView := range chunkViews { + wg.Add(1) + go func(chunkView *filer2.ChunkView) { + defer wg.Done() + + glog.V(3).Infof("read fh reading chunk: %+v", chunkView) + + locations := vid2Locations[volumeId(chunkView.FileId)] + if locations == nil || len(locations.Locations) == 0 { + glog.V(0).Infof("failed to locate %s", chunkView.FileId) + err = fmt.Errorf("failed to locate %s", chunkView.FileId) + return + } + + var n int64 + n, err = util.ReadUrl( + fmt.Sprintf("http://%s/%s", locations.Locations[0].Url, chunkView.FileId), + chunkView.Offset, + int(chunkView.Size), + buff[chunkView.LogicOffset-req.Offset:chunkView.LogicOffset-req.Offset+int64(chunkView.Size)]) + + if err != nil { + + glog.V(0).Infof("%v/%v read http://%s/%v %v bytes: %v", fh.dirPath, fh.name, locations.Locations[0].Url, chunkView.FileId, n, err) + + err = fmt.Errorf("failed to read http://%s/%s: %v", + locations.Locations[0].Url, chunkView.FileId, err) + return + } + + glog.V(3).Infof("read fh read %d bytes: %+v", n, chunkView) + totalRead += n + + }(chunkView) + } + wg.Wait() + + resp.Data = buff[:totalRead] + + return err } // Write to the file handle @@ -179,3 +230,11 @@ func (fh *FileHandle) Flush(ctx context.Context, req *fuse.FlushRequest) error { return err } + +func volumeId(fileId string) string { + lastCommaIndex := strings.LastIndex(fileId, ",") + if lastCommaIndex > 0 { + return fileId[:lastCommaIndex] + } + return fileId +} diff --git a/weed/pb/filer.proto b/weed/pb/filer.proto index b9c3e87f7..455de81d7 100644 --- a/weed/pb/filer.proto +++ b/weed/pb/filer.proto @@ -15,9 +15,6 @@ service SeaweedFiler { rpc GetEntryAttributes (GetEntryAttributesRequest) returns (GetEntryAttributesResponse) { } - rpc GetFileContent (GetFileContentRequest) returns (GetFileContentResponse) { - } - rpc CreateEntry (CreateEntryRequest) returns (CreateEntryResponse) { } @@ -30,6 +27,9 @@ service SeaweedFiler { rpc AssignVolume (AssignVolumeRequest) returns (AssignVolumeResponse) { } + rpc LookupVolume (LookupVolumeRequest) returns (LookupVolumeResponse) { + } + } ////////////////////////////////////////////////// @@ -100,6 +100,13 @@ message CreateEntryRequest { message CreateEntryResponse { } +message UpdateEntryRequest { + string directory = 1; + Entry entry = 2; +} +message UpdateEntryResponse { +} + message DeleteEntryRequest { string directory = 1; string name = 2; @@ -122,9 +129,18 @@ message AssignVolumeResponse { int32 count = 4; } -message UpdateEntryRequest { - string directory = 1; - Entry entry = 2; +message LookupVolumeRequest { + repeated string volume_ids = 1; } -message UpdateEntryResponse { + +message Locations { + repeated Location locations = 1; +} + +message Location { + string url = 1; + string public_url = 2; +} +message LookupVolumeResponse { + map locations_map = 1; } diff --git a/weed/pb/filer_pb/filer.pb.go b/weed/pb/filer_pb/filer.pb.go index b1f0a3e0d..fb0388307 100644 --- a/weed/pb/filer_pb/filer.pb.go +++ b/weed/pb/filer_pb/filer.pb.go @@ -22,12 +22,16 @@ It has these top-level messages: GetFileContentResponse CreateEntryRequest CreateEntryResponse + UpdateEntryRequest + UpdateEntryResponse DeleteEntryRequest DeleteEntryResponse AssignVolumeRequest AssignVolumeResponse - UpdateEntryRequest - UpdateEntryResponse + LookupVolumeRequest + Locations + Location + LookupVolumeResponse */ package filer_pb @@ -371,6 +375,38 @@ func (m *CreateEntryResponse) String() string { return proto.CompactT func (*CreateEntryResponse) ProtoMessage() {} func (*CreateEntryResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{12} } +type UpdateEntryRequest struct { + Directory string `protobuf:"bytes,1,opt,name=directory" json:"directory,omitempty"` + Entry *Entry `protobuf:"bytes,2,opt,name=entry" json:"entry,omitempty"` +} + +func (m *UpdateEntryRequest) Reset() { *m = UpdateEntryRequest{} } +func (m *UpdateEntryRequest) String() string { return proto.CompactTextString(m) } +func (*UpdateEntryRequest) ProtoMessage() {} +func (*UpdateEntryRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{13} } + +func (m *UpdateEntryRequest) GetDirectory() string { + if m != nil { + return m.Directory + } + return "" +} + +func (m *UpdateEntryRequest) GetEntry() *Entry { + if m != nil { + return m.Entry + } + return nil +} + +type UpdateEntryResponse struct { +} + +func (m *UpdateEntryResponse) Reset() { *m = UpdateEntryResponse{} } +func (m *UpdateEntryResponse) String() string { return proto.CompactTextString(m) } +func (*UpdateEntryResponse) ProtoMessage() {} +func (*UpdateEntryResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{14} } + type DeleteEntryRequest struct { Directory string `protobuf:"bytes,1,opt,name=directory" json:"directory,omitempty"` Name string `protobuf:"bytes,2,opt,name=name" json:"name,omitempty"` @@ -380,7 +416,7 @@ type DeleteEntryRequest struct { func (m *DeleteEntryRequest) Reset() { *m = DeleteEntryRequest{} } func (m *DeleteEntryRequest) String() string { return proto.CompactTextString(m) } func (*DeleteEntryRequest) ProtoMessage() {} -func (*DeleteEntryRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{13} } +func (*DeleteEntryRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{15} } func (m *DeleteEntryRequest) GetDirectory() string { if m != nil { @@ -409,7 +445,7 @@ type DeleteEntryResponse struct { func (m *DeleteEntryResponse) Reset() { *m = DeleteEntryResponse{} } func (m *DeleteEntryResponse) String() string { return proto.CompactTextString(m) } func (*DeleteEntryResponse) ProtoMessage() {} -func (*DeleteEntryResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{14} } +func (*DeleteEntryResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{16} } type AssignVolumeRequest struct { Count int32 `protobuf:"varint,1,opt,name=count" json:"count,omitempty"` @@ -420,7 +456,7 @@ type AssignVolumeRequest struct { func (m *AssignVolumeRequest) Reset() { *m = AssignVolumeRequest{} } func (m *AssignVolumeRequest) String() string { return proto.CompactTextString(m) } func (*AssignVolumeRequest) ProtoMessage() {} -func (*AssignVolumeRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{15} } +func (*AssignVolumeRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{17} } func (m *AssignVolumeRequest) GetCount() int32 { if m != nil { @@ -453,7 +489,7 @@ type AssignVolumeResponse struct { func (m *AssignVolumeResponse) Reset() { *m = AssignVolumeResponse{} } func (m *AssignVolumeResponse) String() string { return proto.CompactTextString(m) } func (*AssignVolumeResponse) ProtoMessage() {} -func (*AssignVolumeResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{16} } +func (*AssignVolumeResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{18} } func (m *AssignVolumeResponse) GetFileId() string { if m != nil { @@ -483,37 +519,77 @@ func (m *AssignVolumeResponse) GetCount() int32 { return 0 } -type UpdateEntryRequest struct { - Directory string `protobuf:"bytes,1,opt,name=directory" json:"directory,omitempty"` - Entry *Entry `protobuf:"bytes,2,opt,name=entry" json:"entry,omitempty"` +type LookupVolumeRequest struct { + VolumeIds []string `protobuf:"bytes,1,rep,name=volume_ids,json=volumeIds" json:"volume_ids,omitempty"` } -func (m *UpdateEntryRequest) Reset() { *m = UpdateEntryRequest{} } -func (m *UpdateEntryRequest) String() string { return proto.CompactTextString(m) } -func (*UpdateEntryRequest) ProtoMessage() {} -func (*UpdateEntryRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{17} } +func (m *LookupVolumeRequest) Reset() { *m = LookupVolumeRequest{} } +func (m *LookupVolumeRequest) String() string { return proto.CompactTextString(m) } +func (*LookupVolumeRequest) ProtoMessage() {} +func (*LookupVolumeRequest) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{19} } -func (m *UpdateEntryRequest) GetDirectory() string { +func (m *LookupVolumeRequest) GetVolumeIds() []string { if m != nil { - return m.Directory + return m.VolumeIds } - return "" + return nil } -func (m *UpdateEntryRequest) GetEntry() *Entry { +type Locations struct { + Locations []*Location `protobuf:"bytes,1,rep,name=locations" json:"locations,omitempty"` +} + +func (m *Locations) Reset() { *m = Locations{} } +func (m *Locations) String() string { return proto.CompactTextString(m) } +func (*Locations) ProtoMessage() {} +func (*Locations) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{20} } + +func (m *Locations) GetLocations() []*Location { if m != nil { - return m.Entry + return m.Locations } return nil } -type UpdateEntryResponse struct { +type Location struct { + Url string `protobuf:"bytes,1,opt,name=url" json:"url,omitempty"` + PublicUrl string `protobuf:"bytes,2,opt,name=public_url,json=publicUrl" json:"public_url,omitempty"` } -func (m *UpdateEntryResponse) Reset() { *m = UpdateEntryResponse{} } -func (m *UpdateEntryResponse) String() string { return proto.CompactTextString(m) } -func (*UpdateEntryResponse) ProtoMessage() {} -func (*UpdateEntryResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{18} } +func (m *Location) Reset() { *m = Location{} } +func (m *Location) String() string { return proto.CompactTextString(m) } +func (*Location) ProtoMessage() {} +func (*Location) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{21} } + +func (m *Location) GetUrl() string { + if m != nil { + return m.Url + } + return "" +} + +func (m *Location) GetPublicUrl() string { + if m != nil { + return m.PublicUrl + } + return "" +} + +type LookupVolumeResponse struct { + LocationsMap map[string]*Locations `protobuf:"bytes,1,rep,name=locations_map,json=locationsMap" json:"locations_map,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` +} + +func (m *LookupVolumeResponse) Reset() { *m = LookupVolumeResponse{} } +func (m *LookupVolumeResponse) String() string { return proto.CompactTextString(m) } +func (*LookupVolumeResponse) ProtoMessage() {} +func (*LookupVolumeResponse) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{22} } + +func (m *LookupVolumeResponse) GetLocationsMap() map[string]*Locations { + if m != nil { + return m.LocationsMap + } + return nil +} func init() { proto.RegisterType((*LookupDirectoryEntryRequest)(nil), "filer_pb.LookupDirectoryEntryRequest") @@ -529,12 +605,16 @@ func init() { proto.RegisterType((*GetFileContentResponse)(nil), "filer_pb.GetFileContentResponse") proto.RegisterType((*CreateEntryRequest)(nil), "filer_pb.CreateEntryRequest") proto.RegisterType((*CreateEntryResponse)(nil), "filer_pb.CreateEntryResponse") + proto.RegisterType((*UpdateEntryRequest)(nil), "filer_pb.UpdateEntryRequest") + proto.RegisterType((*UpdateEntryResponse)(nil), "filer_pb.UpdateEntryResponse") proto.RegisterType((*DeleteEntryRequest)(nil), "filer_pb.DeleteEntryRequest") proto.RegisterType((*DeleteEntryResponse)(nil), "filer_pb.DeleteEntryResponse") proto.RegisterType((*AssignVolumeRequest)(nil), "filer_pb.AssignVolumeRequest") proto.RegisterType((*AssignVolumeResponse)(nil), "filer_pb.AssignVolumeResponse") - proto.RegisterType((*UpdateEntryRequest)(nil), "filer_pb.UpdateEntryRequest") - proto.RegisterType((*UpdateEntryResponse)(nil), "filer_pb.UpdateEntryResponse") + proto.RegisterType((*LookupVolumeRequest)(nil), "filer_pb.LookupVolumeRequest") + proto.RegisterType((*Locations)(nil), "filer_pb.Locations") + proto.RegisterType((*Location)(nil), "filer_pb.Location") + proto.RegisterType((*LookupVolumeResponse)(nil), "filer_pb.LookupVolumeResponse") } // Reference imports to suppress errors if they are not otherwise used. @@ -551,11 +631,11 @@ type SeaweedFilerClient interface { LookupDirectoryEntry(ctx context.Context, in *LookupDirectoryEntryRequest, opts ...grpc.CallOption) (*LookupDirectoryEntryResponse, error) ListEntries(ctx context.Context, in *ListEntriesRequest, opts ...grpc.CallOption) (*ListEntriesResponse, error) GetEntryAttributes(ctx context.Context, in *GetEntryAttributesRequest, opts ...grpc.CallOption) (*GetEntryAttributesResponse, error) - GetFileContent(ctx context.Context, in *GetFileContentRequest, opts ...grpc.CallOption) (*GetFileContentResponse, error) CreateEntry(ctx context.Context, in *CreateEntryRequest, opts ...grpc.CallOption) (*CreateEntryResponse, error) UpdateEntry(ctx context.Context, in *UpdateEntryRequest, opts ...grpc.CallOption) (*UpdateEntryResponse, error) DeleteEntry(ctx context.Context, in *DeleteEntryRequest, opts ...grpc.CallOption) (*DeleteEntryResponse, error) AssignVolume(ctx context.Context, in *AssignVolumeRequest, opts ...grpc.CallOption) (*AssignVolumeResponse, error) + LookupVolume(ctx context.Context, in *LookupVolumeRequest, opts ...grpc.CallOption) (*LookupVolumeResponse, error) } type seaweedFilerClient struct { @@ -593,15 +673,6 @@ func (c *seaweedFilerClient) GetEntryAttributes(ctx context.Context, in *GetEntr return out, nil } -func (c *seaweedFilerClient) GetFileContent(ctx context.Context, in *GetFileContentRequest, opts ...grpc.CallOption) (*GetFileContentResponse, error) { - out := new(GetFileContentResponse) - err := grpc.Invoke(ctx, "/filer_pb.SeaweedFiler/GetFileContent", in, out, c.cc, opts...) - if err != nil { - return nil, err - } - return out, nil -} - func (c *seaweedFilerClient) CreateEntry(ctx context.Context, in *CreateEntryRequest, opts ...grpc.CallOption) (*CreateEntryResponse, error) { out := new(CreateEntryResponse) err := grpc.Invoke(ctx, "/filer_pb.SeaweedFiler/CreateEntry", in, out, c.cc, opts...) @@ -638,17 +709,26 @@ func (c *seaweedFilerClient) AssignVolume(ctx context.Context, in *AssignVolumeR return out, nil } +func (c *seaweedFilerClient) LookupVolume(ctx context.Context, in *LookupVolumeRequest, opts ...grpc.CallOption) (*LookupVolumeResponse, error) { + out := new(LookupVolumeResponse) + err := grpc.Invoke(ctx, "/filer_pb.SeaweedFiler/LookupVolume", in, out, c.cc, opts...) + if err != nil { + return nil, err + } + return out, nil +} + // Server API for SeaweedFiler service type SeaweedFilerServer interface { LookupDirectoryEntry(context.Context, *LookupDirectoryEntryRequest) (*LookupDirectoryEntryResponse, error) ListEntries(context.Context, *ListEntriesRequest) (*ListEntriesResponse, error) GetEntryAttributes(context.Context, *GetEntryAttributesRequest) (*GetEntryAttributesResponse, error) - GetFileContent(context.Context, *GetFileContentRequest) (*GetFileContentResponse, error) CreateEntry(context.Context, *CreateEntryRequest) (*CreateEntryResponse, error) UpdateEntry(context.Context, *UpdateEntryRequest) (*UpdateEntryResponse, error) DeleteEntry(context.Context, *DeleteEntryRequest) (*DeleteEntryResponse, error) AssignVolume(context.Context, *AssignVolumeRequest) (*AssignVolumeResponse, error) + LookupVolume(context.Context, *LookupVolumeRequest) (*LookupVolumeResponse, error) } func RegisterSeaweedFilerServer(s *grpc.Server, srv SeaweedFilerServer) { @@ -709,24 +789,6 @@ func _SeaweedFiler_GetEntryAttributes_Handler(srv interface{}, ctx context.Conte return interceptor(ctx, in, info, handler) } -func _SeaweedFiler_GetFileContent_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(GetFileContentRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(SeaweedFilerServer).GetFileContent(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/filer_pb.SeaweedFiler/GetFileContent", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(SeaweedFilerServer).GetFileContent(ctx, req.(*GetFileContentRequest)) - } - return interceptor(ctx, in, info, handler) -} - func _SeaweedFiler_CreateEntry_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { in := new(CreateEntryRequest) if err := dec(in); err != nil { @@ -799,6 +861,24 @@ func _SeaweedFiler_AssignVolume_Handler(srv interface{}, ctx context.Context, de return interceptor(ctx, in, info, handler) } +func _SeaweedFiler_LookupVolume_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(LookupVolumeRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(SeaweedFilerServer).LookupVolume(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/filer_pb.SeaweedFiler/LookupVolume", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(SeaweedFilerServer).LookupVolume(ctx, req.(*LookupVolumeRequest)) + } + return interceptor(ctx, in, info, handler) +} + var _SeaweedFiler_serviceDesc = grpc.ServiceDesc{ ServiceName: "filer_pb.SeaweedFiler", HandlerType: (*SeaweedFilerServer)(nil), @@ -815,10 +895,6 @@ var _SeaweedFiler_serviceDesc = grpc.ServiceDesc{ MethodName: "GetEntryAttributes", Handler: _SeaweedFiler_GetEntryAttributes_Handler, }, - { - MethodName: "GetFileContent", - Handler: _SeaweedFiler_GetFileContent_Handler, - }, { MethodName: "CreateEntry", Handler: _SeaweedFiler_CreateEntry_Handler, @@ -835,6 +911,10 @@ var _SeaweedFiler_serviceDesc = grpc.ServiceDesc{ MethodName: "AssignVolume", Handler: _SeaweedFiler_AssignVolume_Handler, }, + { + MethodName: "LookupVolume", + Handler: _SeaweedFiler_LookupVolume_Handler, + }, }, Streams: []grpc.StreamDesc{}, Metadata: "filer.proto", @@ -843,53 +923,61 @@ var _SeaweedFiler_serviceDesc = grpc.ServiceDesc{ func init() { proto.RegisterFile("filer.proto", fileDescriptor0) } var fileDescriptor0 = []byte{ - // 763 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xb4, 0x56, 0xdd, 0x6e, 0xd3, 0x4a, - 0x10, 0xae, 0xe3, 0x24, 0x6d, 0x26, 0x69, 0xcf, 0xd1, 0x26, 0xed, 0xf1, 0x49, 0x7f, 0x08, 0x86, - 0xa2, 0x22, 0xa4, 0x0a, 0x85, 0x1b, 0x2e, 0xa9, 0xda, 0x52, 0x21, 0x15, 0x55, 0x72, 0x55, 0x24, - 0xae, 0xa2, 0xc4, 0x9e, 0x84, 0x55, 0x1d, 0x3b, 0x78, 0xd7, 0xa0, 0x72, 0x0b, 0xaf, 0xc2, 0x4b, - 0xf0, 0x74, 0x68, 0x7f, 0xe2, 0xac, 0xb1, 0xd3, 0x9f, 0x0b, 0xee, 0x76, 0x67, 0x76, 0xbe, 0xf9, - 0x76, 0x66, 0xbe, 0xb5, 0xa1, 0x39, 0xa6, 0x21, 0x26, 0x87, 0xb3, 0x24, 0xe6, 0x31, 0x59, 0x93, - 0x9b, 0xc1, 0x6c, 0xe4, 0x5e, 0xc0, 0xf6, 0x79, 0x1c, 0x5f, 0xa7, 0xb3, 0x13, 0x9a, 0xa0, 0xcf, - 0xe3, 0xe4, 0xe6, 0x34, 0xe2, 0xc9, 0x8d, 0x87, 0x9f, 0x53, 0x64, 0x9c, 0xec, 0x40, 0x23, 0x98, - 0x3b, 0x1c, 0xab, 0x67, 0x1d, 0x34, 0xbc, 0x85, 0x81, 0x10, 0xa8, 0x46, 0xc3, 0x29, 0x3a, 0x15, - 0xe9, 0x90, 0x6b, 0xf7, 0x14, 0x76, 0xca, 0x01, 0xd9, 0x2c, 0x8e, 0x18, 0x92, 0x7d, 0xa8, 0xa1, - 0x30, 0x48, 0xb4, 0x66, 0xff, 0x9f, 0xc3, 0x39, 0x95, 0x43, 0x75, 0x4e, 0x79, 0xdd, 0x3e, 0x90, - 0x73, 0xca, 0xb8, 0xb0, 0x51, 0x64, 0xf7, 0xa2, 0xe3, 0xbe, 0x81, 0x76, 0x2e, 0x46, 0x67, 0x7c, - 0x0e, 0xab, 0xa8, 0x4c, 0x8e, 0xd5, 0xb3, 0xcb, 0x72, 0xce, 0xfd, 0xee, 0x4f, 0x0b, 0x6a, 0xd2, - 0x94, 0x5d, 0xcd, 0x5a, 0x5c, 0x8d, 0x3c, 0x86, 0x16, 0x65, 0x83, 0x05, 0x01, 0x71, 0xed, 0x35, - 0xaf, 0x49, 0x59, 0x76, 0x55, 0xf2, 0x02, 0xea, 0xfe, 0xa7, 0x34, 0xba, 0x66, 0x8e, 0x2d, 0x53, - 0xb5, 0x17, 0xa9, 0xde, 0xd2, 0x10, 0x8f, 0x85, 0xcf, 0xd3, 0x47, 0xc8, 0x6b, 0x80, 0x21, 0xe7, - 0x09, 0x1d, 0xa5, 0x1c, 0x99, 0x53, 0x95, 0xf5, 0x70, 0x8c, 0x80, 0x94, 0xe1, 0x51, 0xe6, 0xf7, - 0x8c, 0xb3, 0xee, 0x18, 0x1a, 0x19, 0x1c, 0xf9, 0x0f, 0x56, 0x45, 0xcc, 0x80, 0x06, 0x9a, 0x6d, - 0x5d, 0x6c, 0xdf, 0x05, 0x64, 0x0b, 0xea, 0xf1, 0x78, 0xcc, 0x90, 0x4b, 0xa6, 0xb6, 0xa7, 0x77, - 0xe2, 0x6e, 0x8c, 0x7e, 0x43, 0xc7, 0xee, 0x59, 0x07, 0x55, 0x4f, 0xae, 0x49, 0x07, 0x6a, 0x53, - 0x4e, 0xa7, 0x28, 0x69, 0xd8, 0x9e, 0xda, 0xb8, 0x3f, 0x2c, 0xd8, 0xc8, 0xd3, 0x20, 0xdb, 0xd0, - 0x90, 0xd9, 0x24, 0x82, 0x25, 0x11, 0xe4, 0x34, 0x5d, 0xe6, 0x50, 0x2a, 0x06, 0x4a, 0x16, 0x32, - 0x8d, 0x03, 0x95, 0x74, 0x5d, 0x85, 0xbc, 0x8f, 0x03, 0x24, 0xff, 0x82, 0x9d, 0xd2, 0x40, 0xa6, - 0x5d, 0xf7, 0xc4, 0x52, 0x58, 0x26, 0x34, 0x70, 0x6a, 0xca, 0x32, 0xa1, 0x81, 0x3b, 0x81, 0xff, - 0xcf, 0x50, 0xf6, 0xf5, 0xc6, 0x28, 0x88, 0x9e, 0x89, 0xb2, 0x4e, 0xed, 0x02, 0xcc, 0x86, 0x09, - 0x46, 0x5c, 0x74, 0x4b, 0x8f, 0x67, 0x43, 0x59, 0x4e, 0x68, 0x62, 0x56, 0xcc, 0x36, 0x2b, 0xe6, - 0x7e, 0xb7, 0xa0, 0x5b, 0x96, 0x49, 0x4f, 0x52, 0xbe, 0x61, 0xd6, 0xfd, 0x1b, 0x66, 0xcc, 0x45, - 0xe5, 0xce, 0xb9, 0x70, 0x5f, 0xc2, 0xe6, 0x19, 0x72, 0x69, 0x8f, 0x23, 0x8e, 0x11, 0x9f, 0x5f, - 0x75, 0x59, 0xa7, 0xdd, 0x3e, 0x6c, 0xfd, 0x19, 0xa1, 0x29, 0x3b, 0xb0, 0xea, 0x2b, 0x93, 0x0c, - 0x69, 0x79, 0xf3, 0xad, 0xfb, 0x11, 0xc8, 0x71, 0x82, 0x43, 0x8e, 0x0f, 0x10, 0x7c, 0x26, 0xde, - 0xca, 0xad, 0xe2, 0xdd, 0x84, 0x76, 0x0e, 0x5a, 0x71, 0x71, 0x29, 0x90, 0x13, 0x0c, 0xf1, 0x41, - 0x19, 0x4b, 0x9e, 0x98, 0x82, 0x0e, 0xed, 0x82, 0x0e, 0x05, 0x83, 0x5c, 0x2a, 0xcd, 0x60, 0x0a, - 0xed, 0x23, 0xc6, 0xe8, 0x24, 0xfa, 0x10, 0x87, 0xe9, 0x14, 0xe7, 0x14, 0x3a, 0x50, 0xf3, 0xe3, - 0x54, 0x97, 0xa8, 0xe6, 0xa9, 0x0d, 0xd9, 0x03, 0xf0, 0xe3, 0x30, 0x44, 0x9f, 0xd3, 0x38, 0xd2, - 0x04, 0x0c, 0x0b, 0xe9, 0x41, 0x33, 0xc1, 0x59, 0x48, 0xfd, 0xa1, 0x3c, 0xa0, 0x26, 0xc9, 0x34, - 0xb9, 0x5f, 0xa0, 0x93, 0x4f, 0xa7, 0x9b, 0xb2, 0x54, 0xb1, 0x42, 0x0c, 0x49, 0xa8, 0x73, 0x89, - 0xa5, 0x9c, 0xe4, 0x74, 0x14, 0x52, 0x7f, 0x20, 0x1c, 0xb6, 0x9e, 0x64, 0x69, 0xb9, 0x4a, 0xc2, - 0x05, 0xf3, 0xaa, 0xc1, 0x5c, 0xb4, 0xf6, 0x6a, 0x16, 0xfc, 0xad, 0xd6, 0xe6, 0xa0, 0xd5, 0x8d, - 0xfa, 0xbf, 0x6a, 0xd0, 0xba, 0xc4, 0xe1, 0x57, 0xc4, 0x40, 0x4c, 0x61, 0x42, 0x26, 0xd0, 0x29, - 0xfb, 0x0c, 0x90, 0xfd, 0x05, 0xee, 0x2d, 0xdf, 0x9d, 0xee, 0xb3, 0xbb, 0x8e, 0xe9, 0x86, 0xae, - 0x90, 0x73, 0x68, 0x1a, 0x8f, 0x3e, 0xd9, 0x31, 0x02, 0x0b, 0xdf, 0x8f, 0xee, 0xee, 0x12, 0x6f, - 0x86, 0x36, 0x04, 0x52, 0xd4, 0x3f, 0x79, 0xb2, 0x08, 0x5b, 0xfa, 0x0e, 0x75, 0x9f, 0xde, 0x7e, - 0x28, 0x4b, 0x71, 0x05, 0x1b, 0x79, 0xad, 0x92, 0x47, 0xb9, 0xc8, 0xa2, 0xee, 0xbb, 0xbd, 0xe5, - 0x07, 0xcc, 0x3a, 0x18, 0x9a, 0x33, 0xeb, 0x50, 0x54, 0xb9, 0x59, 0x87, 0x32, 0xa1, 0x4a, 0x34, - 0xa3, 0xcd, 0x26, 0x5a, 0x71, 0xb0, 0x4c, 0xb4, 0x92, 0xd9, 0x50, 0x68, 0x86, 0x1a, 0x4d, 0xb4, - 0xe2, 0x7b, 0x60, 0xa2, 0x95, 0x49, 0x78, 0x85, 0x5c, 0x40, 0xcb, 0x54, 0x15, 0x31, 0x02, 0x4a, - 0xc4, 0xdd, 0xdd, 0x5b, 0xe6, 0x9e, 0x03, 0x8e, 0xea, 0xf2, 0xa7, 0xe8, 0xd5, 0xef, 0x00, 0x00, - 0x00, 0xff, 0xff, 0xec, 0x38, 0x88, 0xb2, 0x23, 0x09, 0x00, 0x00, + // 890 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xb4, 0x56, 0x6d, 0x6f, 0xdc, 0x44, + 0x10, 0x8e, 0xcf, 0x71, 0x12, 0xcf, 0x5d, 0x78, 0xd9, 0x4b, 0x8b, 0xb9, 0x26, 0x55, 0x58, 0x28, + 0x6a, 0x85, 0x14, 0x45, 0x81, 0x0f, 0x15, 0x08, 0x89, 0xaa, 0x29, 0x55, 0xa5, 0x54, 0x95, 0x5c, + 0x82, 0xc4, 0xa7, 0x93, 0xcf, 0x9e, 0x3b, 0x56, 0xf1, 0xd9, 0xc6, 0xbb, 0x0e, 0x0a, 0x5f, 0xe1, + 0xaf, 0xf0, 0x0f, 0xf8, 0x07, 0xfc, 0x31, 0xb4, 0x2f, 0xb6, 0xd7, 0xb1, 0xaf, 0x2f, 0x1f, 0xf8, + 0xb6, 0x3b, 0x3b, 0xf3, 0xcc, 0x33, 0xbb, 0x33, 0x8f, 0x0d, 0xe3, 0x25, 0x4b, 0xb1, 0x3c, 0x29, + 0xca, 0x5c, 0xe4, 0x64, 0x4f, 0x6d, 0xe6, 0xc5, 0x82, 0xbe, 0x82, 0x7b, 0x17, 0x79, 0x7e, 0x55, + 0x15, 0xe7, 0xac, 0xc4, 0x58, 0xe4, 0xe5, 0xcd, 0xb3, 0x4c, 0x94, 0x37, 0x21, 0xfe, 0x56, 0x21, + 0x17, 0xe4, 0x10, 0xfc, 0xa4, 0x3e, 0x08, 0x9c, 0x63, 0xe7, 0xa1, 0x1f, 0xb6, 0x06, 0x42, 0x60, + 0x3b, 0x8b, 0xd6, 0x18, 0x8c, 0xd4, 0x81, 0x5a, 0xd3, 0x67, 0x70, 0x38, 0x0c, 0xc8, 0x8b, 0x3c, + 0xe3, 0x48, 0x1e, 0x80, 0x87, 0xd2, 0xa0, 0xd0, 0xc6, 0x67, 0x1f, 0x9e, 0xd4, 0x54, 0x4e, 0xb4, + 0x9f, 0x3e, 0xa5, 0x67, 0x40, 0x2e, 0x18, 0x17, 0xd2, 0xc6, 0x90, 0xbf, 0x13, 0x1d, 0xfa, 0x03, + 0x4c, 0x3b, 0x31, 0x26, 0xe3, 0x23, 0xd8, 0x45, 0x6d, 0x0a, 0x9c, 0x63, 0x77, 0x28, 0x67, 0x7d, + 0x4e, 0xff, 0x76, 0xc0, 0x53, 0xa6, 0xa6, 0x34, 0xa7, 0x2d, 0x8d, 0x7c, 0x06, 0x13, 0xc6, 0xe7, + 0x2d, 0x01, 0x59, 0xf6, 0x5e, 0x38, 0x66, 0xbc, 0x29, 0x95, 0x7c, 0x05, 0x3b, 0xf1, 0xaf, 0x55, + 0x76, 0xc5, 0x03, 0x57, 0xa5, 0x9a, 0xb6, 0xa9, 0x7e, 0x64, 0x29, 0x3e, 0x95, 0x67, 0xa1, 0x71, + 0x21, 0x8f, 0x01, 0x22, 0x21, 0x4a, 0xb6, 0xa8, 0x04, 0xf2, 0x60, 0x5b, 0xdd, 0x47, 0x60, 0x05, + 0x54, 0x1c, 0x9f, 0x34, 0xe7, 0xa1, 0xe5, 0x4b, 0x97, 0xe0, 0x37, 0x70, 0xe4, 0x13, 0xd8, 0x95, + 0x31, 0x73, 0x96, 0x18, 0xb6, 0x3b, 0x72, 0xfb, 0x22, 0x21, 0x77, 0x61, 0x27, 0x5f, 0x2e, 0x39, + 0x0a, 0xc5, 0xd4, 0x0d, 0xcd, 0x4e, 0xd6, 0xc6, 0xd9, 0x1f, 0x18, 0xb8, 0xc7, 0xce, 0xc3, 0xed, + 0x50, 0xad, 0xc9, 0x01, 0x78, 0x6b, 0xc1, 0xd6, 0xa8, 0x68, 0xb8, 0xa1, 0xde, 0xd0, 0xbf, 0x1c, + 0xf8, 0xa0, 0x4b, 0x83, 0xdc, 0x03, 0x5f, 0x65, 0x53, 0x08, 0x8e, 0x42, 0x50, 0xdd, 0xf4, 0xba, + 0x83, 0x32, 0xb2, 0x50, 0x9a, 0x90, 0x75, 0x9e, 0xe8, 0xa4, 0xfb, 0x3a, 0xe4, 0x65, 0x9e, 0x20, + 0xf9, 0x08, 0xdc, 0x8a, 0x25, 0x2a, 0xed, 0x7e, 0x28, 0x97, 0xd2, 0xb2, 0x62, 0x49, 0xe0, 0x69, + 0xcb, 0x8a, 0x25, 0x74, 0x05, 0x9f, 0x3e, 0x47, 0xf5, 0xae, 0x37, 0xd6, 0x85, 0x98, 0x9e, 0x18, + 0x7a, 0xa9, 0x23, 0x80, 0x22, 0x2a, 0x31, 0x13, 0xf2, 0xb5, 0x4c, 0x7b, 0xfa, 0xda, 0x72, 0xce, + 0x4a, 0xfb, 0xc6, 0x5c, 0xfb, 0xc6, 0xe8, 0x9f, 0x0e, 0xcc, 0x86, 0x32, 0x99, 0x4e, 0xea, 0x3e, + 0x98, 0xf3, 0xee, 0x0f, 0x66, 0xf5, 0xc5, 0xe8, 0xad, 0x7d, 0x41, 0x4f, 0xe1, 0xce, 0x73, 0x14, + 0xca, 0x9e, 0x67, 0x02, 0x33, 0x51, 0x97, 0xba, 0xe9, 0xa5, 0xe9, 0x19, 0xdc, 0xbd, 0x1d, 0x61, + 0x28, 0x07, 0xb0, 0x1b, 0x6b, 0x93, 0x0a, 0x99, 0x84, 0xf5, 0x96, 0xfe, 0x02, 0xe4, 0x69, 0x89, + 0x91, 0xc0, 0xf7, 0x18, 0xf8, 0x66, 0x78, 0x47, 0x6f, 0x1c, 0xde, 0x3b, 0x30, 0xed, 0x40, 0x6b, + 0x2e, 0x32, 0xe3, 0x65, 0x91, 0xfc, 0x5f, 0x19, 0x3b, 0xd0, 0x26, 0x23, 0x03, 0x72, 0x8e, 0x29, + 0xbe, 0x57, 0xc6, 0x01, 0x51, 0xeb, 0x4d, 0xbe, 0xdb, 0x9b, 0x7c, 0xc9, 0xa0, 0x93, 0xca, 0x30, + 0x58, 0xc3, 0xf4, 0x09, 0xe7, 0x6c, 0x95, 0xfd, 0x9c, 0xa7, 0xd5, 0x1a, 0x6b, 0x0a, 0x07, 0xe0, + 0xc5, 0x79, 0x65, 0x1e, 0xc5, 0x0b, 0xf5, 0x86, 0xdc, 0x07, 0x88, 0xf3, 0x34, 0xc5, 0x58, 0xb0, + 0x3c, 0x33, 0x04, 0x2c, 0x0b, 0x39, 0x86, 0x71, 0x89, 0x45, 0xca, 0xe2, 0x48, 0x39, 0xe8, 0xde, + 0xb5, 0x4d, 0xf4, 0x1a, 0x0e, 0xba, 0xe9, 0x4c, 0x1b, 0x6c, 0xd4, 0x08, 0x39, 0x7e, 0x65, 0x6a, + 0x72, 0xc9, 0xa5, 0x9a, 0x9d, 0x6a, 0x91, 0xb2, 0x78, 0x2e, 0x0f, 0x5c, 0x33, 0x3b, 0xca, 0x72, + 0x59, 0xa6, 0x2d, 0xf3, 0x6d, 0x8b, 0x39, 0xfd, 0x06, 0xa6, 0x5a, 0xf5, 0xbb, 0x65, 0x1e, 0x01, + 0x5c, 0x2b, 0xc3, 0x9c, 0x25, 0x5a, 0x7d, 0xfd, 0xd0, 0xd7, 0x96, 0x17, 0x09, 0xa7, 0xdf, 0x83, + 0x7f, 0x91, 0x6b, 0xe6, 0x9c, 0x9c, 0x82, 0x9f, 0xd6, 0x1b, 0x23, 0xd4, 0xa4, 0x7d, 0xed, 0xda, + 0x2f, 0x6c, 0x9d, 0xe8, 0x77, 0xb0, 0x57, 0x9b, 0xeb, 0x3a, 0x9c, 0x4d, 0x75, 0x8c, 0x6e, 0xd5, + 0x41, 0xff, 0x75, 0xe0, 0xa0, 0x4b, 0xd9, 0x5c, 0xd5, 0x25, 0xec, 0x37, 0x29, 0xe6, 0xeb, 0xa8, + 0x30, 0x5c, 0x4e, 0x6d, 0x2e, 0xfd, 0xb0, 0x86, 0x20, 0x7f, 0x19, 0x15, 0xba, 0x05, 0x26, 0xa9, + 0x65, 0x9a, 0xfd, 0x04, 0x1f, 0xf7, 0x5c, 0x24, 0xeb, 0x2b, 0xac, 0x7b, 0x50, 0x2e, 0xc9, 0x23, + 0xf0, 0xae, 0xa3, 0xb4, 0x42, 0xd3, 0xef, 0xd3, 0xfe, 0x0d, 0xf0, 0x50, 0x7b, 0x7c, 0x3b, 0x7a, + 0xec, 0x9c, 0xfd, 0xe3, 0xc1, 0xe4, 0x35, 0x46, 0xbf, 0x23, 0x26, 0x72, 0xfa, 0x4b, 0xb2, 0xaa, + 0xab, 0xea, 0x7e, 0x7e, 0xc9, 0x83, 0xdb, 0xf4, 0x07, 0xbf, 0xf7, 0xb3, 0x2f, 0xdf, 0xe6, 0x66, + 0xda, 0x7a, 0x8b, 0x5c, 0xc0, 0xd8, 0xfa, 0xd8, 0x92, 0x43, 0x2b, 0xb0, 0xf7, 0xdd, 0x9e, 0x1d, + 0x6d, 0x38, 0x6d, 0xd0, 0x22, 0x20, 0x7d, 0xdd, 0x25, 0x9f, 0xb7, 0x61, 0x1b, 0xf5, 0x7f, 0xf6, + 0xc5, 0x9b, 0x9d, 0x6c, 0xc2, 0x96, 0x28, 0xd9, 0x84, 0xfb, 0x32, 0x68, 0x13, 0x1e, 0x52, 0x32, + 0x85, 0x66, 0x09, 0x8e, 0x8d, 0xd6, 0x97, 0x38, 0x1b, 0x6d, 0x48, 0xa5, 0x14, 0x9a, 0x25, 0x1e, + 0x36, 0x5a, 0x5f, 0xbe, 0x6c, 0xb4, 0x21, 0xc5, 0xd9, 0x22, 0xaf, 0x60, 0x62, 0x8b, 0x00, 0xb1, + 0x02, 0x06, 0xb4, 0x68, 0x76, 0x7f, 0xd3, 0xb1, 0x0d, 0x68, 0xf7, 0xbc, 0x0d, 0x38, 0x30, 0xf5, + 0x36, 0xe0, 0xd0, 0xa8, 0xd0, 0xad, 0xc5, 0x8e, 0xfa, 0x0d, 0xfd, 0xfa, 0xbf, 0x00, 0x00, 0x00, + 0xff, 0xff, 0x01, 0xeb, 0x13, 0xfa, 0x95, 0x0a, 0x00, 0x00, } diff --git a/weed/server/filer_grpc_server.go b/weed/server/filer_grpc_server.go index da02ce169..7fba6a7ce 100644 --- a/weed/server/filer_grpc_server.go +++ b/weed/server/filer_grpc_server.go @@ -3,7 +3,6 @@ package weed_server import ( "context" "github.com/chrislusf/seaweedfs/weed/operation" - "github.com/chrislusf/seaweedfs/weed/util" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "fmt" "github.com/chrislusf/seaweedfs/weed/filer2" @@ -88,20 +87,31 @@ func (fs *FilerServer) GetEntryAttributes(ctx context.Context, req *filer_pb.Get }, nil } -func (fs *FilerServer) GetFileContent(ctx context.Context, req *filer_pb.GetFileContentRequest) (*filer_pb.GetFileContentResponse, error) { +func (fs *FilerServer) LookupVolume(ctx context.Context, req *filer_pb.LookupVolumeRequest) (*filer_pb.LookupVolumeResponse, error) { - server, err := operation.LookupFileId(fs.getMasterNode(), req.FileId) + lookupResult, err := operation.LookupVolumeIds(fs.getMasterNode(), req.VolumeIds) if err != nil { return nil, err } - content, err := util.Get(server) - if err != nil { - return nil, err + + resp := &filer_pb.LookupVolumeResponse{ + LocationsMap: make(map[string]*filer_pb.Locations), } - return &filer_pb.GetFileContentResponse{ - Content: content, - }, nil + for vid, locations := range lookupResult { + var locs []*filer_pb.Location + for _, loc := range locations.Locations { + locs = append(locs, &filer_pb.Location{ + Url: loc.Url, + PublicUrl: loc.PublicUrl, + }) + } + resp.LocationsMap[vid] = &filer_pb.Locations{ + Locations: locs, + } + } + + return resp, nil } func (fs *FilerServer) CreateEntry(ctx context.Context, req *filer_pb.CreateEntryRequest) (resp *filer_pb.CreateEntryResponse, err error) { diff --git a/weed/util/http_util.go b/weed/util/http_util.go index ca9f7c50e..00dbdf90f 100644 --- a/weed/util/http_util.go +++ b/weed/util/http_util.go @@ -183,3 +183,33 @@ func NormalizeUrl(url string) string { } return "http://" + url } + +func ReadUrl(fileUrl string, offset int64, size int, buf []byte) (n int64, e error) { + + req, _ := http.NewRequest("GET", fileUrl, nil) + req.Header.Add("Range", fmt.Sprintf("bytes=%d-%d", offset, offset+int64(size))) + + r, err := client.Do(req) + if err != nil { + return 0, err + } + defer r.Body.Close() + if r.StatusCode >= 400 { + return 0, fmt.Errorf("%s: %s", fileUrl, r.Status) + } + + var i, m int + + for { + m, err = r.Body.Read(buf[i:cap(buf)]) + i += m + n += int64(m) + if err == io.EOF { + return n, nil + } + if e != nil { + return n, e + } + } + +} From e18c7e160c075b44d2982e452cf4882ba0bfe895 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Thu, 24 May 2018 23:19:56 -0700 Subject: [PATCH 27/83] avoid empty chunk view --- weed/filer2/filechunks.go | 2 +- weed/filer2/filechunks_test.go | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/weed/filer2/filechunks.go b/weed/filer2/filechunks.go index 6bdfbd48e..5877f2f71 100644 --- a/weed/filer2/filechunks.go +++ b/weed/filer2/filechunks.go @@ -66,7 +66,7 @@ func ReadFromChunks(chunks []*filer_pb.FileChunk, offset int64, size int) (views stop := offset + int64(size) for _, chunk := range visibles { - if chunk.start <= offset && offset < chunk.stop { + if chunk.start <= offset && offset < chunk.stop && offset < stop { views = append(views, &ChunkView{ FileId: chunk.fileId, Offset: offset - chunk.start, // offset is the data starting location in this file id diff --git a/weed/filer2/filechunks_test.go b/weed/filer2/filechunks_test.go index 24897215e..033e66b1e 100644 --- a/weed/filer2/filechunks_test.go +++ b/weed/filer2/filechunks_test.go @@ -249,6 +249,20 @@ func TestChunksReading(t *testing.T) { {Offset: 0, Size: 100, FileId: "abc", LogicOffset:0}, }, }, + // case 7: edge cases + { + Chunks: []*filer_pb.FileChunk{ + {Offset: 0, Size: 100, FileId: "abc", Mtime: 123}, + {Offset: 100, Size: 100, FileId: "asdf", Mtime: 134}, + {Offset: 200, Size: 100, FileId: "fsad", Mtime: 353}, + }, + Offset: 0, + Size: 200, + Expected: []*ChunkView{ + {Offset: 0, Size: 100, FileId: "abc", LogicOffset:0}, + {Offset: 0, Size: 100, FileId: "asdf", LogicOffset:100}, + }, + }, } for i, testcase := range testcases { From 9f4f8de9adf3115e08875a6ab4a9c51932fd3d4a Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Thu, 24 May 2018 23:20:12 -0700 Subject: [PATCH 28/83] skip printout --- weed/filesys/file.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/weed/filesys/file.go b/weed/filesys/file.go index d7f0d6bc0..fa3838d3a 100644 --- a/weed/filesys/file.go +++ b/weed/filesys/file.go @@ -2,8 +2,6 @@ package filesys import ( "context" - "fmt" - "bazil.org/fuse" "bazil.org/fuse/fs" "github.com/chrislusf/seaweedfs/weed/glog" @@ -99,8 +97,9 @@ func (file *File) Setattr(ctx context.Context, req *fuse.SetattrRequest, resp *f glog.V(3).Infof("%v file setattr %+v", fullPath, req) if req.Valid.Size() { + glog.V(3).Infof("%v file setattr set size=%v", fullPath, req.Size) if req.Size == 0 { - fmt.Printf("truncate %v \n", fullPath) + // fmt.Printf("truncate %v \n", fullPath) file.Chunks = nil } file.attributes.FileSize = req.Size From 7b81cf37628bfabc4ac6018c4b123dbd245ed158 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Thu, 24 May 2018 23:20:26 -0700 Subject: [PATCH 29/83] better logs --- weed/filesys/filehandle.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/weed/filesys/filehandle.go b/weed/filesys/filehandle.go index c71f1ee36..7ab2513e1 100644 --- a/weed/filesys/filehandle.go +++ b/weed/filesys/filehandle.go @@ -143,9 +143,9 @@ func (fh *FileHandle) Write(ctx context.Context, req *fuse.WriteRequest, resp *f Collection: "", } - glog.V(1).Infof("assign volume: %v", request) resp, err := client.AssignVolume(ctx, request) if err != nil { + glog.V(0).Infof("assign volume failure %v: %v", request, err) return err } @@ -160,9 +160,11 @@ func (fh *FileHandle) Write(ctx context.Context, req *fuse.WriteRequest, resp *f bufReader := bytes.NewReader(req.Data) uploadResult, err := operation.Upload(fileUrl, fh.name, bufReader, false, "application/octet-stream", nil, "") if err != nil { + glog.V(0).Infof("upload data %v to %s: %v", req, fileUrl, err) return fmt.Errorf("upload data: %v", err) } if uploadResult.Error != "" { + glog.V(0).Infof("upload failure %v to %s: %v", req, fileUrl, err) return fmt.Errorf("upload result: %v", uploadResult.Error) } @@ -217,6 +219,9 @@ func (fh *FileHandle) Flush(ctx context.Context, req *fuse.FlushRequest) error { } glog.V(1).Infof("%s/%s set chunks: %v", fh.dirPath, fh.name, len(fh.Chunks)) + for i, chunk := range fh.Chunks { + glog.V(1).Infof("%s/%s chunks %d: %v [%d,%d)", fh.dirPath, fh.name, i, chunk.FileId, chunk.Offset, chunk.Offset+int64(chunk.Size)) + } if _, err := client.UpdateEntry(ctx, request); err != nil { return fmt.Errorf("update fh: %v", err) } From c11d84f31414997ac43c3c1e638fe762318110d8 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Thu, 24 May 2018 23:20:56 -0700 Subject: [PATCH 30/83] fix reading from a url --- weed/util/http_util.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/weed/util/http_util.go b/weed/util/http_util.go index 00dbdf90f..6494041e3 100644 --- a/weed/util/http_util.go +++ b/weed/util/http_util.go @@ -201,7 +201,10 @@ func ReadUrl(fileUrl string, offset int64, size int, buf []byte) (n int64, e err var i, m int for { - m, err = r.Body.Read(buf[i:cap(buf)]) + m, err = r.Body.Read(buf[i:]) + if m == 0 { + return + } i += m n += int64(m) if err == io.EOF { From a01557f4cb98235bbd8d0258ae6f9fcda40f4f9f Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Thu, 24 May 2018 23:21:12 -0700 Subject: [PATCH 31/83] adjust filer web page --- weed/filer2/filer_structure.go | 12 ++++++++++++ weed/server/filer_grpc_server.go | 4 ++-- weed/server/filer_ui/templates.go | 20 ++++++++++++++++---- 3 files changed, 30 insertions(+), 6 deletions(-) diff --git a/weed/filer2/filer_structure.go b/weed/filer2/filer_structure.go index 5e5382d35..c732d9230 100644 --- a/weed/filer2/filer_structure.go +++ b/weed/filer2/filer_structure.go @@ -55,6 +55,18 @@ type Entry struct { Chunks []*filer_pb.FileChunk `json:"chunks,omitempty"` } +func (entry Entry) Size() uint64 { + return TotalSize(entry.Chunks) +} + +func (entry Entry) Timestamp() time.Time { + if entry.IsDirectory() { + return entry.Crtime + } else { + return entry.Mtime + } +} + type AbstractFiler interface { CreateEntry(*Entry) (error) AppendFileChunk(FullPath, []*filer_pb.FileChunk) (err error) diff --git a/weed/server/filer_grpc_server.go b/weed/server/filer_grpc_server.go index 7fba6a7ce..050785be2 100644 --- a/weed/server/filer_grpc_server.go +++ b/weed/server/filer_grpc_server.go @@ -46,7 +46,7 @@ func (fs *FilerServer) ListEntries(ctx context.Context, req *filer_pb.ListEntrie IsDirectory: entry.IsDirectory(), Chunks: entry.Chunks, Attributes: &filer_pb.FuseAttributes{ - FileSize: filer2.TotalSize(entry.Chunks), + FileSize: entry.Size(), Mtime: entry.Mtime.Unix(), Gid: entry.Gid, Uid: entry.Uid, @@ -73,7 +73,7 @@ func (fs *FilerServer) GetEntryAttributes(ctx context.Context, req *filer_pb.Get return nil, fmt.Errorf("file %s not found", fullpath) } - attributes.FileSize = filer2.TotalSize(entry.Chunks) + attributes.FileSize = entry.Size() attributes.FileMode = uint32(entry.Mode) attributes.Uid = entry.Uid attributes.Gid = entry.Gid diff --git a/weed/server/filer_ui/templates.go b/weed/server/filer_ui/templates.go index bba60ff1a..5e901cd68 100644 --- a/weed/server/filer_ui/templates.go +++ b/weed/server/filer_ui/templates.go @@ -24,10 +24,11 @@ var StatusTpl = template.Must(template.New("status").Parse(`
-
{{if .ShouldDisplayLoadMore}} From f8776ad5cdda169ea8627d2df2f6475f9ee40691 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Thu, 24 May 2018 23:42:13 -0700 Subject: [PATCH 32/83] add breadcrumb to filer UI --- weed/server/filer_server_handlers_admin.go | 8 +++++--- weed/server/filer_server_handlers_read.go | 2 ++ weed/server/filer_ui/breadcrumb.go | 24 ++++++++++++++++++++++ weed/server/filer_ui/templates.go | 7 ++++++- 4 files changed, 37 insertions(+), 4 deletions(-) create mode 100644 weed/server/filer_ui/breadcrumb.go diff --git a/weed/server/filer_server_handlers_admin.go b/weed/server/filer_server_handlers_admin.go index 7f17f936d..01b18a76d 100644 --- a/weed/server/filer_server_handlers_admin.go +++ b/weed/server/filer_server_handlers_admin.go @@ -34,9 +34,11 @@ func (fs *FilerServer) registerHandler(w http.ResponseWriter, r *http.Request) { entry := &filer2.Entry{ FullPath: filer2.FullPath(path), Attr: filer2.Attr{ - Mode: 0660, - Uid: uint32(uid), - Gid: uint32(gid), + Mode: 0660, + Crtime: time.Now(), + Mtime: time.Now(), + Uid: uint32(uid), + Gid: uint32(gid), }, Chunks: []*filer_pb.FileChunk{{ FileId: fileId, diff --git a/weed/server/filer_server_handlers_read.go b/weed/server/filer_server_handlers_read.go index 52b7ba177..7c479e18b 100644 --- a/weed/server/filer_server_handlers_read.go +++ b/weed/server/filer_server_handlers_read.go @@ -52,12 +52,14 @@ func (fs *FilerServer) listDirectoryHandler(w http.ResponseWriter, r *http.Reque args := struct { Path string + Breadcrumbs []ui.Breadcrumb Entries interface{} Limit int LastFileName string ShouldDisplayLoadMore bool }{ path, + ui.ToBreadcrumb(path), entries, limit, lastFileName, diff --git a/weed/server/filer_ui/breadcrumb.go b/weed/server/filer_ui/breadcrumb.go new file mode 100644 index 000000000..146b6c6c4 --- /dev/null +++ b/weed/server/filer_ui/breadcrumb.go @@ -0,0 +1,24 @@ +package master_ui + +import ( + "strings" + "path/filepath" +) + +type Breadcrumb struct { + Name string + Link string +} + +func ToBreadcrumb(fullpath string) (crumbs []Breadcrumb) { + parts := strings.Split(fullpath, "/") + + for i := 0; i < len(parts); i++ { + crumbs = append(crumbs, Breadcrumb{ + Name: parts[i] + "/", + Link: "/" + filepath.Join(parts[0:i+1]...), + }) + } + + return +} diff --git a/weed/server/filer_ui/templates.go b/weed/server/filer_ui/templates.go index 5e901cd68..d5c42672e 100644 --- a/weed/server/filer_ui/templates.go +++ b/weed/server/filer_ui/templates.go @@ -20,7 +20,12 @@ var StatusTpl = template.Must(template.New("status").Parse(`
- {{.Path}} + {{ range $entry := .Breadcrumbs }} + + {{ $entry.Name }} + + {{ end }} +
From 0a223838bdee52fd912f5eb6de4720da44d315ac Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Fri, 25 May 2018 00:57:25 -0700 Subject: [PATCH 33/83] refactoring --- weed/filer2/filer_structure.go | 10 ------ weed/filesys/dir.go | 18 +++++----- weed/filesys/file.go | 14 +++----- weed/filesys/filehandle.go | 64 ++++++++++++++++------------------ 4 files changed, 44 insertions(+), 62 deletions(-) diff --git a/weed/filer2/filer_structure.go b/weed/filer2/filer_structure.go index c732d9230..7a5dc3d8d 100644 --- a/weed/filer2/filer_structure.go +++ b/weed/filer2/filer_structure.go @@ -67,16 +67,6 @@ func (entry Entry) Timestamp() time.Time { } } -type AbstractFiler interface { - CreateEntry(*Entry) (error) - AppendFileChunk(FullPath, []*filer_pb.FileChunk) (err error) - FindEntry(FullPath) (found bool, fileEntry *Entry, err error) - DeleteEntry(FullPath) (fileEntry *Entry, err error) - - ListDirectoryEntries(dirPath FullPath) ([]*Entry, error) - UpdateEntry(*Entry) (error) -} - var ErrNotFound = errors.New("filer: no entry is found in filer store") type FilerStore interface { diff --git a/weed/filesys/dir.go b/weed/filesys/dir.go index 8d07705c6..5e9ede79d 100644 --- a/weed/filesys/dir.go +++ b/weed/filesys/dir.go @@ -23,7 +23,11 @@ type Dir struct { } var _ = fs.Node(&Dir{}) +var _ = fs.NodeCreater(&Dir{}) +var _ = fs.NodeMkdirer(&Dir{}) +var _ = fs.NodeStringLookuper(&Dir{}) var _ = fs.HandleReadDirAller(&Dir{}) +var _ = fs.NodeRemover(&Dir{}) func (dir *Dir) Attr(context context.Context, attr *fuse.Attr) error { @@ -117,15 +121,11 @@ func (dir *Dir) Create(ctx context.Context, req *fuse.CreateRequest, file := dir.newFile(req.Name, nil) dir.NodeMap[req.Name] = file return file, &FileHandle{ - wfs: file.wfs, - dirPath: file.dir.Path, - name: file.Name, - RequestId: req.Header.ID, - NodeId: req.Header.Node, - Uid: req.Uid, - Gid: req.Gid, - attributes: file.attributes, - Chunks: file.Chunks, + f: file, + RequestId: req.Header.ID, + NodeId: req.Header.Node, + Uid: req.Uid, + Gid: req.Gid, }, nil } diff --git a/weed/filesys/file.go b/weed/filesys/file.go index fa3838d3a..c8d96c316 100644 --- a/weed/filesys/file.go +++ b/weed/filesys/file.go @@ -78,15 +78,11 @@ func (file *File) Open(ctx context.Context, req *fuse.OpenRequest, resp *fuse.Op glog.V(3).Infof("%v file open %+v", fullPath, req) return &FileHandle{ - wfs: file.wfs, - dirPath: file.dir.Path, - name: file.Name, - RequestId: req.Header.ID, - NodeId: req.Header.Node, - Uid: req.Uid, - Gid: req.Gid, - attributes: file.attributes, - Chunks: file.Chunks, + f: file, + RequestId: req.Header.ID, + NodeId: req.Header.Node, + Uid: req.Uid, + Gid: req.Gid, }, nil } diff --git a/weed/filesys/filehandle.go b/weed/filesys/filehandle.go index 7ab2513e1..414057f4e 100644 --- a/weed/filesys/filehandle.go +++ b/weed/filesys/filehandle.go @@ -24,15 +24,11 @@ type FileHandle struct { handle uint64 - wfs *WFS - dirPath string - name string - RequestId fuse.RequestID // unique ID for request - NodeId fuse.NodeID // file or directory the request is about - Uid uint32 // user ID of process making request - Gid uint32 // group ID of process making request - attributes *filer_pb.FuseAttributes - Chunks []*filer_pb.FileChunk + f *File + RequestId fuse.RequestID // unique ID for request + NodeId fuse.NodeID // file or directory the request is about + Uid uint32 // user ID of process making request + Gid uint32 // group ID of process making request } var _ = fs.Handle(&FileHandle{}) @@ -44,16 +40,16 @@ var _ = fs.HandleReleaser(&FileHandle{}) func (fh *FileHandle) Read(ctx context.Context, req *fuse.ReadRequest, resp *fuse.ReadResponse) error { - glog.V(3).Infof("%v/%v read fh: [%d,%d)", fh.dirPath, fh.name, req.Offset, req.Offset+int64(req.Size)) + glog.V(3).Infof("%v/%v read fh: [%d,%d)", fh.f.dir.Path, fh.f.Name, req.Offset, req.Offset+int64(req.Size)) - if len(fh.Chunks) == 0 { - glog.V(0).Infof("empty fh %v/%v", fh.dirPath, fh.name) - return fmt.Errorf("empty file %v/%v", fh.dirPath, fh.name) + if len(fh.f.Chunks) == 0 { + glog.V(0).Infof("empty fh %v/%v", fh.f.dir.Path, fh.f.Name) + return fmt.Errorf("empty file %v/%v", fh.f.dir.Path, fh.f.Name) } buff := make([]byte, req.Size) - chunkViews := filer2.ReadFromChunks(fh.Chunks, req.Offset, req.Size) + chunkViews := filer2.ReadFromChunks(fh.f.Chunks, req.Offset, req.Size) var vids []string for _, chunkView := range chunkViews { @@ -62,7 +58,7 @@ func (fh *FileHandle) Read(ctx context.Context, req *fuse.ReadRequest, resp *fus vid2Locations := make(map[string]*filer_pb.Locations) - err := fh.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error { + err := fh.f.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error { glog.V(4).Infof("read fh lookup volume id locations: %v", vids) resp, err := client.LookupVolume(ctx, &filer_pb.LookupVolumeRequest{ @@ -78,7 +74,7 @@ func (fh *FileHandle) Read(ctx context.Context, req *fuse.ReadRequest, resp *fus }) if err != nil { - glog.V(3).Infof("%v/%v read fh lookup volume ids: %v", fh.dirPath, fh.name, err) + glog.V(3).Infof("%v/%v read fh lookup volume ids: %v", fh.f.dir.Path, fh.f.Name, err) return fmt.Errorf("failed to lookup volume ids %v: %v", vids, err) } @@ -107,7 +103,7 @@ func (fh *FileHandle) Read(ctx context.Context, req *fuse.ReadRequest, resp *fus if err != nil { - glog.V(0).Infof("%v/%v read http://%s/%v %v bytes: %v", fh.dirPath, fh.name, locations.Locations[0].Url, chunkView.FileId, n, err) + glog.V(0).Infof("%v/%v read http://%s/%v %v bytes: %v", fh.f.dir.Path, fh.f.Name, locations.Locations[0].Url, chunkView.FileId, n, err) err = fmt.Errorf("failed to read http://%s/%s: %v", locations.Locations[0].Url, chunkView.FileId, err) @@ -131,11 +127,11 @@ func (fh *FileHandle) Write(ctx context.Context, req *fuse.WriteRequest, resp *f // write the request to volume servers - glog.V(3).Infof("%+v/%v write fh: %+v", fh.dirPath, fh.name, req) + glog.V(3).Infof("%+v/%v write fh: %+v", fh.f.dir.Path, fh.f.Name, req) var fileId, host string - if err := fh.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error { + if err := fh.f.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error { request := &filer_pb.AssignVolumeRequest{ Count: 1, @@ -158,7 +154,7 @@ func (fh *FileHandle) Write(ctx context.Context, req *fuse.WriteRequest, resp *f fileUrl := fmt.Sprintf("http://%s/%s", host, fileId) bufReader := bytes.NewReader(req.Data) - uploadResult, err := operation.Upload(fileUrl, fh.name, bufReader, false, "application/octet-stream", nil, "") + uploadResult, err := operation.Upload(fileUrl, fh.f.Name, bufReader, false, "application/octet-stream", nil, "") if err != nil { glog.V(0).Infof("upload data %v to %s: %v", req, fileUrl, err) return fmt.Errorf("upload data: %v", err) @@ -170,14 +166,14 @@ func (fh *FileHandle) Write(ctx context.Context, req *fuse.WriteRequest, resp *f resp.Size = int(uploadResult.Size) - fh.Chunks = append(fh.Chunks, &filer_pb.FileChunk{ + fh.f.Chunks = append(fh.f.Chunks, &filer_pb.FileChunk{ FileId: fileId, Offset: req.Offset, Size: uint64(uploadResult.Size), Mtime: time.Now().UnixNano(), }) - glog.V(1).Infof("uploaded %s/%s to: %v, [%d,%d)", fh.dirPath, fh.name, fileUrl, req.Offset, req.Offset+int64(resp.Size)) + glog.V(1).Infof("uploaded %s/%s to: %v, [%d,%d)", fh.f.dir.Path, fh.f.Name, fileUrl, req.Offset, req.Offset+int64(resp.Size)) fh.dirty = true @@ -186,7 +182,7 @@ func (fh *FileHandle) Write(ctx context.Context, req *fuse.WriteRequest, resp *f func (fh *FileHandle) Release(ctx context.Context, req *fuse.ReleaseRequest) error { - glog.V(3).Infof("%+v/%v release fh", fh.dirPath, fh.name) + glog.V(3).Infof("%+v/%v release fh", fh.f.dir.Path, fh.f.Name) return nil } @@ -196,31 +192,31 @@ func (fh *FileHandle) Release(ctx context.Context, req *fuse.ReleaseRequest) err func (fh *FileHandle) Flush(ctx context.Context, req *fuse.FlushRequest) error { // fflush works at fh level // send the data to the OS - glog.V(3).Infof("%s/%s fh flush %v", fh.dirPath, fh.name, req) + glog.V(3).Infof("%s/%s fh flush %v", fh.f.dir.Path, fh.f.Name, req) if !fh.dirty { return nil } - if len(fh.Chunks) == 0 { - glog.V(2).Infof("fh %s/%s flush skipping empty: %v", fh.dirPath, fh.name, req) + if len(fh.f.Chunks) == 0 { + glog.V(2).Infof("fh %s/%s flush skipping empty: %v", fh.f.dir.Path, fh.f.Name, req) return nil } - err := fh.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error { + err := fh.f.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error { request := &filer_pb.UpdateEntryRequest{ - Directory: fh.dirPath, + Directory: fh.f.dir.Path, Entry: &filer_pb.Entry{ - Name: fh.name, - Attributes: fh.attributes, - Chunks: fh.Chunks, + Name: fh.f.Name, + Attributes: fh.f.attributes, + Chunks: fh.f.Chunks, }, } - glog.V(1).Infof("%s/%s set chunks: %v", fh.dirPath, fh.name, len(fh.Chunks)) - for i, chunk := range fh.Chunks { - glog.V(1).Infof("%s/%s chunks %d: %v [%d,%d)", fh.dirPath, fh.name, i, chunk.FileId, chunk.Offset, chunk.Offset+int64(chunk.Size)) + glog.V(1).Infof("%s/%s set chunks: %v", fh.f.dir.Path, fh.f.Name, len(fh.f.Chunks)) + for i, chunk := range fh.f.Chunks { + glog.V(1).Infof("%s/%s chunks %d: %v [%d,%d)", fh.f.dir.Path, fh.f.Name, i, chunk.FileId, chunk.Offset, chunk.Offset+int64(chunk.Size)) } if _, err := client.UpdateEntry(ctx, request); err != nil { return fmt.Errorf("update fh: %v", err) From 6d1bcd4b8c9a0579d703ecae3d6d3f06bdff644b Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Fri, 25 May 2018 01:22:31 -0700 Subject: [PATCH 34/83] use existing attributes instead of fetching from filer --- weed/filesys/file.go | 53 +++++++++++++++++++++++--------------------- 1 file changed, 28 insertions(+), 25 deletions(-) diff --git a/weed/filesys/file.go b/weed/filesys/file.go index c8d96c316..09ae4c871 100644 --- a/weed/filesys/file.go +++ b/weed/filesys/file.go @@ -28,36 +28,39 @@ type File struct { func (file *File) Attr(context context.Context, attr *fuse.Attr) error { fullPath := filepath.Join(file.dir.Path, file.Name) - item := file.wfs.listDirectoryEntriesCache.Get(fullPath) - if item != nil { - entry := item.Value().(*filer_pb.Entry) - file.Chunks = entry.Chunks - file.attributes = entry.Attributes - glog.V(1).Infof("file attr read cached %v attributes", file.Name) - } else { - err := file.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error { - - request := &filer_pb.GetEntryAttributesRequest{ - Name: file.Name, - ParentDir: file.dir.Path, - } - resp, err := client.GetEntryAttributes(context, request) - if err != nil { - glog.V(0).Infof("file attr read file %v: %v", request, err) - return err - } + if file.attributes == nil { + item := file.wfs.listDirectoryEntriesCache.Get(fullPath) + if item != nil { + entry := item.Value().(*filer_pb.Entry) + file.Chunks = entry.Chunks + file.attributes = entry.Attributes + glog.V(1).Infof("file attr read cached %v attributes", file.Name) + } else { + err := file.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error { + + request := &filer_pb.GetEntryAttributesRequest{ + Name: file.Name, + ParentDir: file.dir.Path, + } - file.attributes = resp.Attributes - file.Chunks = resp.Chunks + resp, err := client.GetEntryAttributes(context, request) + if err != nil { + glog.V(0).Infof("file attr read file %v: %v", request, err) + return err + } - glog.V(1).Infof("file attr %v %+v: %d", fullPath, file.attributes, filer2.TotalSize(file.Chunks)) + file.attributes = resp.Attributes + file.Chunks = resp.Chunks - return nil - }) + glog.V(1).Infof("file attr %v %+v: %d", fullPath, file.attributes, filer2.TotalSize(file.Chunks)) - if err != nil { - return err + return nil + }) + + if err != nil { + return err + } } } From ac66c133a5a5fff9c9815a27b20d07926e214b8d Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Fri, 25 May 2018 01:27:21 -0700 Subject: [PATCH 35/83] do not read attributes when file is opened --- weed/filesys/dir.go | 1 + weed/filesys/file.go | 5 ++++- weed/filesys/filehandle.go | 2 ++ 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/weed/filesys/dir.go b/weed/filesys/dir.go index 5e9ede79d..ba9f84654 100644 --- a/weed/filesys/dir.go +++ b/weed/filesys/dir.go @@ -120,6 +120,7 @@ func (dir *Dir) Create(ctx context.Context, req *fuse.CreateRequest, if err == nil { file := dir.newFile(req.Name, nil) dir.NodeMap[req.Name] = file + file.isOpen = true return file, &FileHandle{ f: file, RequestId: req.Header.ID, diff --git a/weed/filesys/file.go b/weed/filesys/file.go index 09ae4c871..d63f9d62d 100644 --- a/weed/filesys/file.go +++ b/weed/filesys/file.go @@ -23,13 +23,14 @@ type File struct { dir *Dir wfs *WFS attributes *filer_pb.FuseAttributes + isOpen bool } func (file *File) Attr(context context.Context, attr *fuse.Attr) error { fullPath := filepath.Join(file.dir.Path, file.Name) - if file.attributes == nil { + if file.attributes == nil || !file.isOpen { item := file.wfs.listDirectoryEntriesCache.Get(fullPath) if item != nil { entry := item.Value().(*filer_pb.Entry) @@ -80,6 +81,8 @@ func (file *File) Open(ctx context.Context, req *fuse.OpenRequest, resp *fuse.Op glog.V(3).Infof("%v file open %+v", fullPath, req) + file.isOpen = true + return &FileHandle{ f: file, RequestId: req.Header.ID, diff --git a/weed/filesys/filehandle.go b/weed/filesys/filehandle.go index 414057f4e..d9dc6795e 100644 --- a/weed/filesys/filehandle.go +++ b/weed/filesys/filehandle.go @@ -184,6 +184,8 @@ func (fh *FileHandle) Release(ctx context.Context, req *fuse.ReleaseRequest) err glog.V(3).Infof("%+v/%v release fh", fh.f.dir.Path, fh.f.Name) + fh.f.isOpen = false + return nil } From 72fc001d0dfc73b5cbe390fa815c9f92d07e728e Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Fri, 25 May 2018 01:30:16 -0700 Subject: [PATCH 36/83] remove unused code --- weed/filer2/embedded/embedded_store.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/weed/filer2/embedded/embedded_store.go b/weed/filer2/embedded/embedded_store.go index bca182f36..f2cafacdd 100644 --- a/weed/filer2/embedded/embedded_store.go +++ b/weed/filer2/embedded/embedded_store.go @@ -21,10 +21,6 @@ func (filer *EmbeddedStore) InsertEntry(entry *filer2.Entry) (err error) { return nil } -func (filer *EmbeddedStore) AddDirectoryLink(directory *filer2.Entry, delta int32) (err error) { - return nil -} - func (filer *EmbeddedStore) UpdateEntry(entry *filer2.Entry) (err error) { return nil } From 6de84c64c67eb0e1161ad1b627917d466d49e6c6 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Fri, 25 May 2018 23:26:40 -0700 Subject: [PATCH 37/83] adding create time --- weed/filesys/dir.go | 2 + weed/pb/filer.proto | 1 + weed/pb/filer_pb/filer.pb.go | 123 +++++++++++++++++-------------- weed/server/filer_grpc_server.go | 2 + 4 files changed, 71 insertions(+), 57 deletions(-) diff --git a/weed/filesys/dir.go b/weed/filesys/dir.go index ba9f84654..6406dff95 100644 --- a/weed/filesys/dir.go +++ b/weed/filesys/dir.go @@ -102,6 +102,7 @@ func (dir *Dir) Create(ctx context.Context, req *fuse.CreateRequest, IsDirectory: req.Mode&os.ModeDir > 0, Attributes: &filer_pb.FuseAttributes{ Mtime: time.Now().Unix(), + Crtime: time.Now().Unix(), FileMode: uint32(req.Mode), Uid: req.Uid, Gid: req.Gid, @@ -146,6 +147,7 @@ func (dir *Dir) Mkdir(ctx context.Context, req *fuse.MkdirRequest) (fs.Node, err IsDirectory: true, Attributes: &filer_pb.FuseAttributes{ Mtime: time.Now().Unix(), + Crtime: time.Now().Unix(), FileMode: uint32(req.Mode), Uid: req.Uid, Gid: req.Gid, diff --git a/weed/pb/filer.proto b/weed/pb/filer.proto index 455de81d7..98c39d4b2 100644 --- a/weed/pb/filer.proto +++ b/weed/pb/filer.proto @@ -71,6 +71,7 @@ message FuseAttributes { uint32 file_mode = 3; uint32 uid = 4; uint32 gid = 5; + int64 crtime = 6; } message GetEntryAttributesRequest { diff --git a/weed/pb/filer_pb/filer.pb.go b/weed/pb/filer_pb/filer.pb.go index fb0388307..0473e77c0 100644 --- a/weed/pb/filer_pb/filer.pb.go +++ b/weed/pb/filer_pb/filer.pb.go @@ -213,6 +213,7 @@ type FuseAttributes struct { FileMode uint32 `protobuf:"varint,3,opt,name=file_mode,json=fileMode" json:"file_mode,omitempty"` Uid uint32 `protobuf:"varint,4,opt,name=uid" json:"uid,omitempty"` Gid uint32 `protobuf:"varint,5,opt,name=gid" json:"gid,omitempty"` + Crtime int64 `protobuf:"varint,6,opt,name=crtime" json:"crtime,omitempty"` } func (m *FuseAttributes) Reset() { *m = FuseAttributes{} } @@ -255,6 +256,13 @@ func (m *FuseAttributes) GetGid() uint32 { return 0 } +func (m *FuseAttributes) GetCrtime() int64 { + if m != nil { + return m.Crtime + } + return 0 +} + type GetEntryAttributesRequest struct { Name string `protobuf:"bytes,1,opt,name=name" json:"name,omitempty"` ParentDir string `protobuf:"bytes,2,opt,name=parent_dir,json=parentDir" json:"parent_dir,omitempty"` @@ -923,61 +931,62 @@ var _SeaweedFiler_serviceDesc = grpc.ServiceDesc{ func init() { proto.RegisterFile("filer.proto", fileDescriptor0) } var fileDescriptor0 = []byte{ - // 890 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xb4, 0x56, 0x6d, 0x6f, 0xdc, 0x44, - 0x10, 0x8e, 0xcf, 0x71, 0x12, 0xcf, 0x5d, 0x78, 0xd9, 0x4b, 0x8b, 0xb9, 0x26, 0x55, 0x58, 0x28, - 0x6a, 0x85, 0x14, 0x45, 0x81, 0x0f, 0x15, 0x08, 0x89, 0xaa, 0x29, 0x55, 0xa5, 0x54, 0x95, 0x5c, - 0x82, 0xc4, 0xa7, 0x93, 0xcf, 0x9e, 0x3b, 0x56, 0xf1, 0xd9, 0xc6, 0xbb, 0x0e, 0x0a, 0x5f, 0xe1, - 0xaf, 0xf0, 0x0f, 0xf8, 0x07, 0xfc, 0x31, 0xb4, 0x2f, 0xb6, 0xd7, 0xb1, 0xaf, 0x2f, 0x1f, 0xf8, - 0xb6, 0x3b, 0x3b, 0xf3, 0xcc, 0x33, 0xbb, 0x33, 0x8f, 0x0d, 0xe3, 0x25, 0x4b, 0xb1, 0x3c, 0x29, - 0xca, 0x5c, 0xe4, 0x64, 0x4f, 0x6d, 0xe6, 0xc5, 0x82, 0xbe, 0x82, 0x7b, 0x17, 0x79, 0x7e, 0x55, - 0x15, 0xe7, 0xac, 0xc4, 0x58, 0xe4, 0xe5, 0xcd, 0xb3, 0x4c, 0x94, 0x37, 0x21, 0xfe, 0x56, 0x21, - 0x17, 0xe4, 0x10, 0xfc, 0xa4, 0x3e, 0x08, 0x9c, 0x63, 0xe7, 0xa1, 0x1f, 0xb6, 0x06, 0x42, 0x60, - 0x3b, 0x8b, 0xd6, 0x18, 0x8c, 0xd4, 0x81, 0x5a, 0xd3, 0x67, 0x70, 0x38, 0x0c, 0xc8, 0x8b, 0x3c, - 0xe3, 0x48, 0x1e, 0x80, 0x87, 0xd2, 0xa0, 0xd0, 0xc6, 0x67, 0x1f, 0x9e, 0xd4, 0x54, 0x4e, 0xb4, - 0x9f, 0x3e, 0xa5, 0x67, 0x40, 0x2e, 0x18, 0x17, 0xd2, 0xc6, 0x90, 0xbf, 0x13, 0x1d, 0xfa, 0x03, - 0x4c, 0x3b, 0x31, 0x26, 0xe3, 0x23, 0xd8, 0x45, 0x6d, 0x0a, 0x9c, 0x63, 0x77, 0x28, 0x67, 0x7d, - 0x4e, 0xff, 0x76, 0xc0, 0x53, 0xa6, 0xa6, 0x34, 0xa7, 0x2d, 0x8d, 0x7c, 0x06, 0x13, 0xc6, 0xe7, - 0x2d, 0x01, 0x59, 0xf6, 0x5e, 0x38, 0x66, 0xbc, 0x29, 0x95, 0x7c, 0x05, 0x3b, 0xf1, 0xaf, 0x55, - 0x76, 0xc5, 0x03, 0x57, 0xa5, 0x9a, 0xb6, 0xa9, 0x7e, 0x64, 0x29, 0x3e, 0x95, 0x67, 0xa1, 0x71, - 0x21, 0x8f, 0x01, 0x22, 0x21, 0x4a, 0xb6, 0xa8, 0x04, 0xf2, 0x60, 0x5b, 0xdd, 0x47, 0x60, 0x05, - 0x54, 0x1c, 0x9f, 0x34, 0xe7, 0xa1, 0xe5, 0x4b, 0x97, 0xe0, 0x37, 0x70, 0xe4, 0x13, 0xd8, 0x95, - 0x31, 0x73, 0x96, 0x18, 0xb6, 0x3b, 0x72, 0xfb, 0x22, 0x21, 0x77, 0x61, 0x27, 0x5f, 0x2e, 0x39, - 0x0a, 0xc5, 0xd4, 0x0d, 0xcd, 0x4e, 0xd6, 0xc6, 0xd9, 0x1f, 0x18, 0xb8, 0xc7, 0xce, 0xc3, 0xed, - 0x50, 0xad, 0xc9, 0x01, 0x78, 0x6b, 0xc1, 0xd6, 0xa8, 0x68, 0xb8, 0xa1, 0xde, 0xd0, 0xbf, 0x1c, - 0xf8, 0xa0, 0x4b, 0x83, 0xdc, 0x03, 0x5f, 0x65, 0x53, 0x08, 0x8e, 0x42, 0x50, 0xdd, 0xf4, 0xba, - 0x83, 0x32, 0xb2, 0x50, 0x9a, 0x90, 0x75, 0x9e, 0xe8, 0xa4, 0xfb, 0x3a, 0xe4, 0x65, 0x9e, 0x20, - 0xf9, 0x08, 0xdc, 0x8a, 0x25, 0x2a, 0xed, 0x7e, 0x28, 0x97, 0xd2, 0xb2, 0x62, 0x49, 0xe0, 0x69, - 0xcb, 0x8a, 0x25, 0x74, 0x05, 0x9f, 0x3e, 0x47, 0xf5, 0xae, 0x37, 0xd6, 0x85, 0x98, 0x9e, 0x18, - 0x7a, 0xa9, 0x23, 0x80, 0x22, 0x2a, 0x31, 0x13, 0xf2, 0xb5, 0x4c, 0x7b, 0xfa, 0xda, 0x72, 0xce, - 0x4a, 0xfb, 0xc6, 0x5c, 0xfb, 0xc6, 0xe8, 0x9f, 0x0e, 0xcc, 0x86, 0x32, 0x99, 0x4e, 0xea, 0x3e, - 0x98, 0xf3, 0xee, 0x0f, 0x66, 0xf5, 0xc5, 0xe8, 0xad, 0x7d, 0x41, 0x4f, 0xe1, 0xce, 0x73, 0x14, - 0xca, 0x9e, 0x67, 0x02, 0x33, 0x51, 0x97, 0xba, 0xe9, 0xa5, 0xe9, 0x19, 0xdc, 0xbd, 0x1d, 0x61, - 0x28, 0x07, 0xb0, 0x1b, 0x6b, 0x93, 0x0a, 0x99, 0x84, 0xf5, 0x96, 0xfe, 0x02, 0xe4, 0x69, 0x89, - 0x91, 0xc0, 0xf7, 0x18, 0xf8, 0x66, 0x78, 0x47, 0x6f, 0x1c, 0xde, 0x3b, 0x30, 0xed, 0x40, 0x6b, - 0x2e, 0x32, 0xe3, 0x65, 0x91, 0xfc, 0x5f, 0x19, 0x3b, 0xd0, 0x26, 0x23, 0x03, 0x72, 0x8e, 0x29, - 0xbe, 0x57, 0xc6, 0x01, 0x51, 0xeb, 0x4d, 0xbe, 0xdb, 0x9b, 0x7c, 0xc9, 0xa0, 0x93, 0xca, 0x30, - 0x58, 0xc3, 0xf4, 0x09, 0xe7, 0x6c, 0x95, 0xfd, 0x9c, 0xa7, 0xd5, 0x1a, 0x6b, 0x0a, 0x07, 0xe0, - 0xc5, 0x79, 0x65, 0x1e, 0xc5, 0x0b, 0xf5, 0x86, 0xdc, 0x07, 0x88, 0xf3, 0x34, 0xc5, 0x58, 0xb0, - 0x3c, 0x33, 0x04, 0x2c, 0x0b, 0x39, 0x86, 0x71, 0x89, 0x45, 0xca, 0xe2, 0x48, 0x39, 0xe8, 0xde, - 0xb5, 0x4d, 0xf4, 0x1a, 0x0e, 0xba, 0xe9, 0x4c, 0x1b, 0x6c, 0xd4, 0x08, 0x39, 0x7e, 0x65, 0x6a, - 0x72, 0xc9, 0xa5, 0x9a, 0x9d, 0x6a, 0x91, 0xb2, 0x78, 0x2e, 0x0f, 0x5c, 0x33, 0x3b, 0xca, 0x72, - 0x59, 0xa6, 0x2d, 0xf3, 0x6d, 0x8b, 0x39, 0xfd, 0x06, 0xa6, 0x5a, 0xf5, 0xbb, 0x65, 0x1e, 0x01, - 0x5c, 0x2b, 0xc3, 0x9c, 0x25, 0x5a, 0x7d, 0xfd, 0xd0, 0xd7, 0x96, 0x17, 0x09, 0xa7, 0xdf, 0x83, - 0x7f, 0x91, 0x6b, 0xe6, 0x9c, 0x9c, 0x82, 0x9f, 0xd6, 0x1b, 0x23, 0xd4, 0xa4, 0x7d, 0xed, 0xda, - 0x2f, 0x6c, 0x9d, 0xe8, 0x77, 0xb0, 0x57, 0x9b, 0xeb, 0x3a, 0x9c, 0x4d, 0x75, 0x8c, 0x6e, 0xd5, - 0x41, 0xff, 0x75, 0xe0, 0xa0, 0x4b, 0xd9, 0x5c, 0xd5, 0x25, 0xec, 0x37, 0x29, 0xe6, 0xeb, 0xa8, - 0x30, 0x5c, 0x4e, 0x6d, 0x2e, 0xfd, 0xb0, 0x86, 0x20, 0x7f, 0x19, 0x15, 0xba, 0x05, 0x26, 0xa9, - 0x65, 0x9a, 0xfd, 0x04, 0x1f, 0xf7, 0x5c, 0x24, 0xeb, 0x2b, 0xac, 0x7b, 0x50, 0x2e, 0xc9, 0x23, - 0xf0, 0xae, 0xa3, 0xb4, 0x42, 0xd3, 0xef, 0xd3, 0xfe, 0x0d, 0xf0, 0x50, 0x7b, 0x7c, 0x3b, 0x7a, - 0xec, 0x9c, 0xfd, 0xe3, 0xc1, 0xe4, 0x35, 0x46, 0xbf, 0x23, 0x26, 0x72, 0xfa, 0x4b, 0xb2, 0xaa, - 0xab, 0xea, 0x7e, 0x7e, 0xc9, 0x83, 0xdb, 0xf4, 0x07, 0xbf, 0xf7, 0xb3, 0x2f, 0xdf, 0xe6, 0x66, - 0xda, 0x7a, 0x8b, 0x5c, 0xc0, 0xd8, 0xfa, 0xd8, 0x92, 0x43, 0x2b, 0xb0, 0xf7, 0xdd, 0x9e, 0x1d, - 0x6d, 0x38, 0x6d, 0xd0, 0x22, 0x20, 0x7d, 0xdd, 0x25, 0x9f, 0xb7, 0x61, 0x1b, 0xf5, 0x7f, 0xf6, - 0xc5, 0x9b, 0x9d, 0x6c, 0xc2, 0x96, 0x28, 0xd9, 0x84, 0xfb, 0x32, 0x68, 0x13, 0x1e, 0x52, 0x32, - 0x85, 0x66, 0x09, 0x8e, 0x8d, 0xd6, 0x97, 0x38, 0x1b, 0x6d, 0x48, 0xa5, 0x14, 0x9a, 0x25, 0x1e, - 0x36, 0x5a, 0x5f, 0xbe, 0x6c, 0xb4, 0x21, 0xc5, 0xd9, 0x22, 0xaf, 0x60, 0x62, 0x8b, 0x00, 0xb1, - 0x02, 0x06, 0xb4, 0x68, 0x76, 0x7f, 0xd3, 0xb1, 0x0d, 0x68, 0xf7, 0xbc, 0x0d, 0x38, 0x30, 0xf5, - 0x36, 0xe0, 0xd0, 0xa8, 0xd0, 0xad, 0xc5, 0x8e, 0xfa, 0x0d, 0xfd, 0xfa, 0xbf, 0x00, 0x00, 0x00, - 0xff, 0xff, 0x01, 0xeb, 0x13, 0xfa, 0x95, 0x0a, 0x00, 0x00, + // 899 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xb4, 0x56, 0xdd, 0x6e, 0xdc, 0x44, + 0x14, 0x8e, 0xd7, 0xf1, 0x26, 0x3e, 0xbb, 0xe1, 0x67, 0x36, 0x2d, 0x66, 0x9b, 0x54, 0x61, 0xa0, + 0xa8, 0x15, 0x52, 0x14, 0x05, 0x2e, 0x2a, 0x10, 0x12, 0x55, 0x53, 0xaa, 0x4a, 0xa9, 0x2a, 0xb9, + 0x04, 0x89, 0xab, 0x95, 0x63, 0x9f, 0x5d, 0x46, 0xf1, 0xda, 0xc6, 0x33, 0x0e, 0x0a, 0xb7, 0xbc, + 0x06, 0xb7, 0xbc, 0x01, 0x6f, 0xc0, 0x8b, 0xa1, 0xf9, 0xb1, 0x3d, 0x8e, 0xbd, 0xfd, 0xb9, 0xe0, + 0x6e, 0xe6, 0xfc, 0x7c, 0xe7, 0x3b, 0x33, 0x67, 0x3e, 0x1b, 0x26, 0x4b, 0x96, 0x62, 0x79, 0x5c, + 0x94, 0xb9, 0xc8, 0xc9, 0xae, 0xda, 0x2c, 0x8a, 0x4b, 0xfa, 0x0a, 0xee, 0x9d, 0xe7, 0xf9, 0x55, + 0x55, 0x9c, 0xb1, 0x12, 0x63, 0x91, 0x97, 0x37, 0xcf, 0x32, 0x51, 0xde, 0x84, 0xf8, 0x5b, 0x85, + 0x5c, 0x90, 0x03, 0xf0, 0x93, 0xda, 0x11, 0x38, 0x47, 0xce, 0x43, 0x3f, 0x6c, 0x0d, 0x84, 0xc0, + 0x76, 0x16, 0xad, 0x31, 0x18, 0x29, 0x87, 0x5a, 0xd3, 0x67, 0x70, 0x30, 0x0c, 0xc8, 0x8b, 0x3c, + 0xe3, 0x48, 0x1e, 0x80, 0x87, 0xd2, 0xa0, 0xd0, 0x26, 0xa7, 0x1f, 0x1e, 0xd7, 0x54, 0x8e, 0x75, + 0x9c, 0xf6, 0xd2, 0x53, 0x20, 0xe7, 0x8c, 0x0b, 0x69, 0x63, 0xc8, 0xdf, 0x89, 0x0e, 0xfd, 0x01, + 0x66, 0x9d, 0x1c, 0x53, 0xf1, 0x11, 0xec, 0xa0, 0x36, 0x05, 0xce, 0x91, 0x3b, 0x54, 0xb3, 0xf6, + 0xd3, 0xbf, 0x1d, 0xf0, 0x94, 0xa9, 0x69, 0xcd, 0x69, 0x5b, 0x23, 0x9f, 0xc1, 0x94, 0xf1, 0x45, + 0x4b, 0x40, 0xb6, 0xbd, 0x1b, 0x4e, 0x18, 0x6f, 0x5a, 0x25, 0x5f, 0xc1, 0x38, 0xfe, 0xb5, 0xca, + 0xae, 0x78, 0xe0, 0xaa, 0x52, 0xb3, 0xb6, 0xd4, 0x8f, 0x2c, 0xc5, 0xa7, 0xd2, 0x17, 0x9a, 0x10, + 0xf2, 0x18, 0x20, 0x12, 0xa2, 0x64, 0x97, 0x95, 0x40, 0x1e, 0x6c, 0xab, 0xf3, 0x08, 0xac, 0x84, + 0x8a, 0xe3, 0x93, 0xc6, 0x1f, 0x5a, 0xb1, 0x74, 0x09, 0x7e, 0x03, 0x47, 0x3e, 0x81, 0x1d, 0x99, + 0xb3, 0x60, 0x89, 0x61, 0x3b, 0x96, 0xdb, 0x17, 0x09, 0xb9, 0x0b, 0xe3, 0x7c, 0xb9, 0xe4, 0x28, + 0x14, 0x53, 0x37, 0x34, 0x3b, 0xd9, 0x1b, 0x67, 0x7f, 0x60, 0xe0, 0x1e, 0x39, 0x0f, 0xb7, 0x43, + 0xb5, 0x26, 0xfb, 0xe0, 0xad, 0x05, 0x5b, 0xa3, 0xa2, 0xe1, 0x86, 0x7a, 0x43, 0xff, 0x72, 0xe0, + 0x83, 0x2e, 0x0d, 0x72, 0x0f, 0x7c, 0x55, 0x4d, 0x21, 0x38, 0x0a, 0x41, 0x4d, 0xd3, 0xeb, 0x0e, + 0xca, 0xc8, 0x42, 0x69, 0x52, 0xd6, 0x79, 0xa2, 0x8b, 0xee, 0xe9, 0x94, 0x97, 0x79, 0x82, 0xe4, + 0x23, 0x70, 0x2b, 0x96, 0xa8, 0xb2, 0x7b, 0xa1, 0x5c, 0x4a, 0xcb, 0x8a, 0x25, 0x81, 0xa7, 0x2d, + 0x2b, 0xa6, 0x1a, 0x89, 0x4b, 0x85, 0x3b, 0xd6, 0x8d, 0xe8, 0x1d, 0x5d, 0xc1, 0xa7, 0xcf, 0x51, + 0xdd, 0xf7, 0x8d, 0x75, 0x50, 0x66, 0x56, 0x86, 0x6e, 0xf0, 0x10, 0xa0, 0x88, 0x4a, 0xcc, 0x84, + 0xbc, 0x45, 0x33, 0xb6, 0xbe, 0xb6, 0x9c, 0xb1, 0xd2, 0x3e, 0x49, 0xd7, 0x3e, 0x49, 0xfa, 0xa7, + 0x03, 0xf3, 0xa1, 0x4a, 0x66, 0xc2, 0xba, 0x17, 0xe9, 0xbc, 0xfb, 0x45, 0x5a, 0xf3, 0x32, 0x7a, + 0xeb, 0xbc, 0xd0, 0x13, 0xb8, 0xf3, 0x1c, 0x85, 0xb2, 0xe7, 0x99, 0xc0, 0x4c, 0xd4, 0xad, 0x6e, + 0x9a, 0x00, 0x7a, 0x0a, 0x77, 0x6f, 0x67, 0x18, 0xca, 0x01, 0xec, 0xc4, 0xda, 0xa4, 0x52, 0xa6, + 0x61, 0xbd, 0xa5, 0xbf, 0x00, 0x79, 0x5a, 0x62, 0x24, 0xf0, 0x3d, 0x84, 0xa0, 0x79, 0xd4, 0xa3, + 0x37, 0x3e, 0xea, 0x3b, 0x30, 0xeb, 0x40, 0x6b, 0x2e, 0xb2, 0xe2, 0x45, 0x91, 0xfc, 0x5f, 0x15, + 0x3b, 0xd0, 0xa6, 0x22, 0x03, 0x72, 0x86, 0x29, 0xbe, 0x57, 0xc5, 0x01, 0xb1, 0xeb, 0x29, 0x82, + 0xdb, 0x53, 0x04, 0xc9, 0xa0, 0x53, 0xca, 0x30, 0x58, 0xc3, 0xec, 0x09, 0xe7, 0x6c, 0x95, 0xfd, + 0x9c, 0xa7, 0xd5, 0x1a, 0x6b, 0x0a, 0xfb, 0xe0, 0xc5, 0x79, 0x65, 0x2e, 0xc5, 0x0b, 0xf5, 0x86, + 0xdc, 0x07, 0x88, 0xf3, 0x34, 0xc5, 0x58, 0xb0, 0x3c, 0x33, 0x04, 0x2c, 0x0b, 0x39, 0x82, 0x49, + 0x89, 0x45, 0xca, 0xe2, 0x48, 0x05, 0xe8, 0xd9, 0xb5, 0x4d, 0xf4, 0x1a, 0xf6, 0xbb, 0xe5, 0xcc, + 0x18, 0x6c, 0xd4, 0x0e, 0xf9, 0x2c, 0xcb, 0xd4, 0xd4, 0x92, 0x4b, 0xf5, 0x76, 0xaa, 0xcb, 0x94, + 0xc5, 0x0b, 0xe9, 0x70, 0xcd, 0xdb, 0x51, 0x96, 0x8b, 0x32, 0x6d, 0x99, 0x6f, 0x5b, 0xcc, 0xe9, + 0x37, 0x30, 0xd3, 0x5f, 0x83, 0x6e, 0x9b, 0x87, 0x00, 0xd7, 0xca, 0xb0, 0x60, 0x89, 0x56, 0x65, + 0x3f, 0xf4, 0xb5, 0xe5, 0x45, 0xc2, 0xe9, 0xf7, 0xe0, 0x9f, 0xe7, 0x9a, 0x39, 0x27, 0x27, 0xe0, + 0xa7, 0xf5, 0xc6, 0x08, 0x38, 0x69, 0x6f, 0xbb, 0x8e, 0x0b, 0xdb, 0x20, 0xfa, 0x1d, 0xec, 0xd6, + 0xe6, 0xba, 0x0f, 0x67, 0x53, 0x1f, 0xa3, 0x5b, 0x7d, 0xd0, 0x7f, 0x1d, 0xd8, 0xef, 0x52, 0x36, + 0x47, 0x75, 0x01, 0x7b, 0x4d, 0x89, 0xc5, 0x3a, 0x2a, 0x0c, 0x97, 0x13, 0x9b, 0x4b, 0x3f, 0xad, + 0x21, 0xc8, 0x5f, 0x46, 0x85, 0x1e, 0x81, 0x69, 0x6a, 0x99, 0xe6, 0x3f, 0xc1, 0xc7, 0xbd, 0x10, + 0xc9, 0xfa, 0x0a, 0xeb, 0x19, 0x94, 0x4b, 0xf2, 0x08, 0xbc, 0xeb, 0x28, 0xad, 0xd0, 0xcc, 0xfb, + 0xac, 0x7f, 0x02, 0x3c, 0xd4, 0x11, 0xdf, 0x8e, 0x1e, 0x3b, 0xa7, 0xff, 0x78, 0x30, 0x7d, 0x8d, + 0xd1, 0xef, 0x88, 0x89, 0x7c, 0xfd, 0x25, 0x59, 0xd5, 0x5d, 0x75, 0x3f, 0xcb, 0xe4, 0xc1, 0x6d, + 0xfa, 0x83, 0xff, 0x01, 0xf3, 0x2f, 0xdf, 0x16, 0x66, 0xc6, 0x7a, 0x8b, 0x9c, 0xc3, 0xc4, 0xfa, + 0x08, 0x93, 0x03, 0x2b, 0xb1, 0xf7, 0x3d, 0x9f, 0x1f, 0x6e, 0xf0, 0x36, 0x68, 0x11, 0x90, 0xbe, + 0xee, 0x92, 0xcf, 0xdb, 0xb4, 0x8d, 0xfa, 0x3f, 0xff, 0xe2, 0xcd, 0x41, 0x36, 0x61, 0x4b, 0x94, + 0x6c, 0xc2, 0x7d, 0x19, 0xb4, 0x09, 0x0f, 0x29, 0x99, 0x42, 0xb3, 0x04, 0xc7, 0x46, 0xeb, 0x4b, + 0x9c, 0x8d, 0x36, 0xa4, 0x52, 0x0a, 0xcd, 0x12, 0x0f, 0x1b, 0xad, 0x2f, 0x5f, 0x36, 0xda, 0x90, + 0xe2, 0x6c, 0x91, 0x57, 0x30, 0xb5, 0x45, 0x80, 0x58, 0x09, 0x03, 0x5a, 0x34, 0xbf, 0xbf, 0xc9, + 0x6d, 0x03, 0xda, 0x33, 0x6f, 0x03, 0x0e, 0xbc, 0x7a, 0x1b, 0x70, 0xe8, 0xa9, 0xd0, 0xad, 0xcb, + 0xb1, 0xfa, 0x3d, 0xfd, 0xfa, 0xbf, 0x00, 0x00, 0x00, 0xff, 0xff, 0x83, 0x92, 0xde, 0x64, 0xad, + 0x0a, 0x00, 0x00, } diff --git a/weed/server/filer_grpc_server.go b/weed/server/filer_grpc_server.go index 050785be2..0d1d7ae22 100644 --- a/weed/server/filer_grpc_server.go +++ b/weed/server/filer_grpc_server.go @@ -48,6 +48,7 @@ func (fs *FilerServer) ListEntries(ctx context.Context, req *filer_pb.ListEntrie Attributes: &filer_pb.FuseAttributes{ FileSize: entry.Size(), Mtime: entry.Mtime.Unix(), + Crtime: entry.Crtime.Unix(), Gid: entry.Gid, Uid: entry.Uid, FileMode: uint32(entry.Mode), @@ -78,6 +79,7 @@ func (fs *FilerServer) GetEntryAttributes(ctx context.Context, req *filer_pb.Get attributes.Uid = entry.Uid attributes.Gid = entry.Gid attributes.Mtime = entry.Mtime.Unix() + attributes.Crtime = entry.Crtime.Unix() glog.V(3).Infof("GetEntryAttributes %v size %d chunks %d: %+v", fullpath, attributes.FileSize, len(entry.Chunks), attributes) From c34feca59c6ebfdf7ffd8b24b1c827a2f18f263b Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Fri, 25 May 2018 23:27:06 -0700 Subject: [PATCH 38/83] refactoring --- weed/filer2/entry.go | 42 ++++++++++++++++++ weed/filer2/entry_codec.go | 43 ++++++++++++++++++ weed/filer2/filer_structure.go | 79 ---------------------------------- weed/filer2/filerstore.go | 13 ++++++ weed/filer2/fullpath.go | 31 +++++++++++++ 5 files changed, 129 insertions(+), 79 deletions(-) create mode 100644 weed/filer2/entry.go create mode 100644 weed/filer2/entry_codec.go delete mode 100644 weed/filer2/filer_structure.go create mode 100644 weed/filer2/filerstore.go create mode 100644 weed/filer2/fullpath.go diff --git a/weed/filer2/entry.go b/weed/filer2/entry.go new file mode 100644 index 000000000..53d069ed1 --- /dev/null +++ b/weed/filer2/entry.go @@ -0,0 +1,42 @@ +package filer2 + +import ( + "os" + "time" + + "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" +) + +type Attr struct { + Mtime time.Time // time of last modification + Crtime time.Time // time of creation (OS X only) + Mode os.FileMode // file mode + Uid uint32 // owner uid + Gid uint32 // group gid +} + +func (attr Attr) IsDirectory() (bool) { + return attr.Mode&os.ModeDir > 0 +} + +type Entry struct { + FullPath + + Attr + + // the following is for files + Chunks []*filer_pb.FileChunk `json:"chunks,omitempty"` +} + +func (entry Entry) Size() uint64 { + return TotalSize(entry.Chunks) +} + +func (entry Entry) Timestamp() time.Time { + if entry.IsDirectory() { + return entry.Crtime + } else { + return entry.Mtime + } +} + diff --git a/weed/filer2/entry_codec.go b/weed/filer2/entry_codec.go new file mode 100644 index 000000000..7d2b2da37 --- /dev/null +++ b/weed/filer2/entry_codec.go @@ -0,0 +1,43 @@ +package filer2 + +import ( + "os" + "time" + + "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" + "github.com/gogo/protobuf/proto" + "fmt" +) + +func (entry Entry) EncodeAttributesAndChunks() ([]byte, error) { + message := &filer_pb.Entry{ + Attributes: &filer_pb.FuseAttributes{ + Crtime: entry.Attr.Crtime.Unix(), + Mtime: entry.Attr.Mtime.Unix(), + FileMode: uint32(entry.Attr.Mode), + Uid: entry.Uid, + Gid: entry.Gid, + }, + Chunks: entry.Chunks, + } + return proto.Marshal(message) +} + +func (entry Entry) DecodeAttributesAndChunks(blob []byte) (error) { + + message := &filer_pb.Entry{} + + if err := proto.UnmarshalMerge(blob, message); err != nil { + return fmt.Errorf("decoding value blob for %s: %v", entry.FullPath, err) + } + + entry.Attr.Crtime = time.Unix(message.Attributes.Crtime, 0) + entry.Attr.Mtime = time.Unix(message.Attributes.Mtime, 0) + entry.Attr.Mode = os.FileMode(message.Attributes.FileMode) + entry.Attr.Uid = message.Attributes.Uid + entry.Attr.Gid = message.Attributes.Gid + + entry.Chunks = message.Chunks + + return nil +} diff --git a/weed/filer2/filer_structure.go b/weed/filer2/filer_structure.go deleted file mode 100644 index 7a5dc3d8d..000000000 --- a/weed/filer2/filer_structure.go +++ /dev/null @@ -1,79 +0,0 @@ -package filer2 - -import ( - "errors" - "os" - "time" - "path/filepath" - "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" - "strings" -) - -type FullPath string - -func NewFullPath(dir, name string) FullPath { - if strings.HasSuffix(dir, "/") { - return FullPath(dir + name) - } - return FullPath(dir + "/" + name) -} - -func (fp FullPath) DirAndName() (string, string) { - dir, name := filepath.Split(string(fp)) - if dir == "/" { - return dir, name - } - if len(dir) < 1 { - return "/", "" - } - return dir[:len(dir)-1], name -} - -func (fp FullPath) Name() (string) { - _, name := filepath.Split(string(fp)) - return name -} - -type Attr struct { - Mtime time.Time // time of last modification - Crtime time.Time // time of creation (OS X only) - Mode os.FileMode // file mode - Uid uint32 // owner uid - Gid uint32 // group gid -} - -func (attr Attr) IsDirectory() (bool) { - return attr.Mode&os.ModeDir > 0 -} - -type Entry struct { - FullPath - - Attr - - // the following is for files - Chunks []*filer_pb.FileChunk `json:"chunks,omitempty"` -} - -func (entry Entry) Size() uint64 { - return TotalSize(entry.Chunks) -} - -func (entry Entry) Timestamp() time.Time { - if entry.IsDirectory() { - return entry.Crtime - } else { - return entry.Mtime - } -} - -var ErrNotFound = errors.New("filer: no entry is found in filer store") - -type FilerStore interface { - InsertEntry(*Entry) (error) - UpdateEntry(*Entry) (err error) - FindEntry(FullPath) (found bool, entry *Entry, err error) - DeleteEntry(FullPath) (fileEntry *Entry, err error) - - ListDirectoryEntries(dirPath FullPath, startFileName string, inclusive bool, limit int) ([]*Entry, error) -} diff --git a/weed/filer2/filerstore.go b/weed/filer2/filerstore.go new file mode 100644 index 000000000..e90ec15ed --- /dev/null +++ b/weed/filer2/filerstore.go @@ -0,0 +1,13 @@ +package filer2 + +import "errors" + +type FilerStore interface { + InsertEntry(*Entry) (error) + UpdateEntry(*Entry) (err error) + FindEntry(FullPath) (found bool, entry *Entry, err error) + DeleteEntry(FullPath) (fileEntry *Entry, err error) + ListDirectoryEntries(dirPath FullPath, startFileName string, inclusive bool, limit int) ([]*Entry, error) +} + +var ErrNotFound = errors.New("filer: no entry is found in filer store") diff --git a/weed/filer2/fullpath.go b/weed/filer2/fullpath.go new file mode 100644 index 000000000..20e42e9b9 --- /dev/null +++ b/weed/filer2/fullpath.go @@ -0,0 +1,31 @@ +package filer2 + +import ( + "path/filepath" + "strings" +) + +type FullPath string + +func NewFullPath(dir, name string) FullPath { + if strings.HasSuffix(dir, "/") { + return FullPath(dir + name) + } + return FullPath(dir + "/" + name) +} + +func (fp FullPath) DirAndName() (string, string) { + dir, name := filepath.Split(string(fp)) + if dir == "/" { + return dir, name + } + if len(dir) < 1 { + return "/", "" + } + return dir[:len(dir)-1], name +} + +func (fp FullPath) Name() (string) { + _, name := filepath.Split(string(fp)) + return name +} From 9e77563c990693f22075a20335c7303dbd91953a Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sat, 26 May 2018 03:49:46 -0700 Subject: [PATCH 39/83] add leveldb store MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1. switch to viper for filer store configuration 2. simplify FindEntry() return values, removing “found” 3. add leveldb store --- weed/command/filer.go | 35 +---- weed/command/server.go | 16 -- weed/filer2/configuration.go | 87 +++++++++++ weed/filer2/embedded/embedded_store.go | 38 ----- weed/filer2/entry.go | 5 +- weed/filer2/entry_codec.go | 4 +- weed/filer2/filer.go | 12 +- weed/filer2/filerstore.go | 9 +- weed/filer2/leveldb/leveldb_store.go | 171 +++++++++++++++++++++ weed/filer2/leveldb/leveldb_store_test.go | 61 ++++++++ weed/filer2/memdb/memdb_store.go | 25 +-- weed/filer2/memdb/memdb_store_test.go | 8 +- weed/server/filer_grpc_server.go | 21 +-- weed/server/filer_server.go | 62 +------- weed/server/filer_server_handlers_admin.go | 2 +- weed/server/filer_server_handlers_read.go | 8 +- weed/server/filer_server_handlers_write.go | 11 +- 17 files changed, 382 insertions(+), 193 deletions(-) create mode 100644 weed/filer2/configuration.go delete mode 100644 weed/filer2/embedded/embedded_store.go create mode 100644 weed/filer2/leveldb/leveldb_store.go create mode 100644 weed/filer2/leveldb/leveldb_store_test.go diff --git a/weed/command/filer.go b/weed/command/filer.go index a2929d0d3..5f0ceab48 100644 --- a/weed/command/filer.go +++ b/weed/command/filer.go @@ -2,7 +2,6 @@ package command import ( "net/http" - "os" "strconv" "time" @@ -13,6 +12,7 @@ import ( "google.golang.org/grpc/reflection" "google.golang.org/grpc" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" + "github.com/chrislusf/seaweedfs/weed/filer2" ) var ( @@ -26,17 +26,10 @@ type FilerOptions struct { publicPort *int collection *string defaultReplicaPlacement *string - dir *string redirectOnRead *bool disableDirListing *bool - confFile *string maxMB *int secretKey *string - cassandra_server *string - cassandra_keyspace *string - redis_server *string - redis_password *string - redis_database *int } func init() { @@ -46,23 +39,15 @@ func init() { f.ip = cmdFiler.Flag.String("ip", "", "filer server http listen ip address") f.port = cmdFiler.Flag.Int("port", 8888, "filer server http listen port") f.publicPort = cmdFiler.Flag.Int("port.public", 0, "port opened to public") - f.dir = cmdFiler.Flag.String("dir", os.TempDir(), "directory to store meta data") f.defaultReplicaPlacement = cmdFiler.Flag.String("defaultReplicaPlacement", "000", "default replication type if not specified") f.redirectOnRead = cmdFiler.Flag.Bool("redirectOnRead", false, "whether proxy or redirect to volume server during file GET request") f.disableDirListing = cmdFiler.Flag.Bool("disableDirListing", false, "turn off directory listing") - f.confFile = cmdFiler.Flag.String("confFile", "", "json encoded filer conf file") f.maxMB = cmdFiler.Flag.Int("maxMB", 32, "split files larger than the limit") - f.cassandra_server = cmdFiler.Flag.String("cassandra.server", "", "host[:port] of the cassandra server") - f.cassandra_keyspace = cmdFiler.Flag.String("cassandra.keyspace", "seaweed", "keyspace of the cassandra server") - f.redis_server = cmdFiler.Flag.String("redis.server", "", "comma separated host:port[,host2:port2]* of the redis server, e.g., 127.0.0.1:6379") - f.redis_password = cmdFiler.Flag.String("redis.password", "", "password in clear text") - f.redis_database = cmdFiler.Flag.Int("redis.database", 0, "the database on the redis server") f.secretKey = cmdFiler.Flag.String("secure.secret", "", "secret to encrypt Json Web Token(JWT)") - } var cmdFiler = &Command{ - UsageLine: "filer -port=8888 -dir=/tmp -master=", + UsageLine: "filer -port=8888 -master=", Short: "start a file server that points to a master server", Long: `start a file server which accepts REST operation for any files. @@ -75,20 +60,15 @@ var cmdFiler = &Command{ //return a json format subdirectory and files listing GET /path/to/ - Current mapping metadata store is local embedded leveldb. - It should be highly scalable to hundreds of millions of files on a modest machine. + The configuration file "filer.toml" is read from ".", "$HOME/.seaweedfs/", or "/etc/seaweedfs/", in that order. - Future we will ensure it can avoid of being SPOF. + The following are example filer.toml configuration file. - `, +` + filer2.FILER_TOML_EXAMPLE + "\n", } func runFiler(cmd *Command, args []string) bool { - if err := util.TestFolderWritable(*f.dir); err != nil { - glog.Fatalf("Check Meta Folder (-dir) Writable %s : %s", *f.dir, err) - } - f.start() return true @@ -104,13 +84,10 @@ func (fo *FilerOptions) start() { } fs, nfs_err := weed_server.NewFilerServer(defaultMux, publicVolumeMux, - *fo.ip, *fo.port, *fo.master, *fo.dir, *fo.collection, + *fo.ip, *fo.port, *fo.master, *fo.collection, *fo.defaultReplicaPlacement, *fo.redirectOnRead, *fo.disableDirListing, - *fo.confFile, *fo.maxMB, *fo.secretKey, - *fo.cassandra_server, *fo.cassandra_keyspace, - *fo.redis_server, *fo.redis_password, *fo.redis_database, ) if nfs_err != nil { glog.Fatalf("Filer startup error: %v", nfs_err) diff --git a/weed/command/server.go b/weed/command/server.go index 2425532ac..503b2c61d 100644 --- a/weed/command/server.go +++ b/weed/command/server.go @@ -87,17 +87,10 @@ func init() { filerOptions.collection = cmdServer.Flag.String("filer.collection", "", "all data will be stored in this collection") filerOptions.port = cmdServer.Flag.Int("filer.port", 8888, "filer server http listen port") filerOptions.publicPort = cmdServer.Flag.Int("filer.port.public", 0, "filer server public http listen port") - filerOptions.dir = cmdServer.Flag.String("filer.dir", "", "directory to store meta data, default to a 'filer' sub directory of what -dir is specified") filerOptions.defaultReplicaPlacement = cmdServer.Flag.String("filer.defaultReplicaPlacement", "", "Default replication type if not specified during runtime.") filerOptions.redirectOnRead = cmdServer.Flag.Bool("filer.redirectOnRead", false, "whether proxy or redirect to volume server during file GET request") filerOptions.disableDirListing = cmdServer.Flag.Bool("filer.disableDirListing", false, "turn off directory listing") - filerOptions.confFile = cmdServer.Flag.String("filer.confFile", "", "json encoded filer conf file") filerOptions.maxMB = cmdServer.Flag.Int("filer.maxMB", 32, "split files larger than the limit") - filerOptions.cassandra_server = cmdServer.Flag.String("filer.cassandra.server", "", "host[:port] of the cassandra server") - filerOptions.cassandra_keyspace = cmdServer.Flag.String("filer.cassandra.keyspace", "seaweed", "keyspace of the cassandra server") - filerOptions.redis_server = cmdServer.Flag.String("filer.redis.server", "", "host:port of the redis server, e.g., 127.0.0.1:6379") - filerOptions.redis_password = cmdServer.Flag.String("filer.redis.password", "", "redis password in clear text") - filerOptions.redis_database = cmdServer.Flag.Int("filer.redis.database", 0, "the database on the redis server") } func runServer(cmd *Command, args []string) bool { @@ -157,15 +150,6 @@ func runServer(cmd *Command, args []string) bool { if *masterMetaFolder == "" { *masterMetaFolder = folders[0] } - if *isStartingFiler { - if *filerOptions.dir == "" { - *filerOptions.dir = *masterMetaFolder + "/filer" - os.MkdirAll(*filerOptions.dir, 0700) - } - if err := util.TestFolderWritable(*filerOptions.dir); err != nil { - glog.Fatalf("Check Mapping Meta Folder (-filer.dir=\"%s\") Writable: %s", *filerOptions.dir, err) - } - } if err := util.TestFolderWritable(*masterMetaFolder); err != nil { glog.Fatalf("Check Meta Folder (-mdir=\"%s\") Writable: %s", *masterMetaFolder, err) } diff --git a/weed/filer2/configuration.go b/weed/filer2/configuration.go new file mode 100644 index 000000000..6dabf10b4 --- /dev/null +++ b/weed/filer2/configuration.go @@ -0,0 +1,87 @@ +package filer2 + +import ( + "os" + + "github.com/spf13/viper" + "github.com/chrislusf/seaweedfs/weed/glog" +) + +const ( + FILER_TOML_EXAMPLE = ` +# A sample TOML config file for SeaweedFS filer store + +# local in memory, mostly for testing purpose +[memory] +enabled = false + +[leveldb] +enabled = false +dir = "." # directory to store level db files + +[mysql] +enabled = true +server = "192.168.1.1" +port = 8080 +username = "" +password = "" +database = "" +connection_max_idle = 100 +connection_max_open = 100 + +[postgres] +enabled = false +server = "192.168.1.1" +port = 8080 +username = "" +password = "" +database = "" +connection_max_idle = 100 +connection_max_open = 100 + +` +) + +var ( + Stores []FilerStore +) + +func (f *Filer) LoadConfiguration() { + + // find a filer store + viper.SetConfigName("filer") // name of config file (without extension) + viper.AddConfigPath(".") // optionally look for config in the working directory + viper.AddConfigPath("$HOME/.seaweedfs") // call multiple times to add many search paths + viper.AddConfigPath("/etc/seaweedfs/") // path to look for the config file in + if err := viper.ReadInConfig(); err != nil { // Handle errors reading the config file + glog.Fatalf("Failed to load filer.toml file from current directory, or $HOME/.seaweedfs/, or /etc/seaweedfs/" + + "\n\nPlease follow this example and add a filer.toml file to " + + "current directory, or $HOME/.seaweedfs/, or /etc/seaweedfs/:\n" + FILER_TOML_EXAMPLE) + } + + glog.V(0).Infof("Reading filer configuration from %s", viper.ConfigFileUsed()) + for _, store := range Stores { + if viper.GetBool(store.GetName() + ".enabled") { + viperSub := viper.Sub(store.GetName()) + if err := store.Initialize(viperSub); err != nil { + glog.Fatalf("Failed to initialize store for %s: %+v", + store.GetName(), err) + } + f.SetStore(store) + glog.V(0).Infof("Configure filer for %s from %s", store.GetName(), viper.ConfigFileUsed()) + return + } + } + + println() + println("Supported filer stores are:") + for _, store := range Stores { + println(" " + store.GetName()) + } + + println() + println("Please configure a supported filer store in", viper.ConfigFileUsed()) + println() + + os.Exit(-1) +} diff --git a/weed/filer2/embedded/embedded_store.go b/weed/filer2/embedded/embedded_store.go deleted file mode 100644 index f2cafacdd..000000000 --- a/weed/filer2/embedded/embedded_store.go +++ /dev/null @@ -1,38 +0,0 @@ -package embedded - -import ( - "github.com/syndtr/goleveldb/leveldb" - "github.com/chrislusf/seaweedfs/weed/filer2" -) - -type EmbeddedStore struct { - db *leveldb.DB -} - -func NewEmbeddedStore(dir string) (filer *EmbeddedStore, err error) { - filer = &EmbeddedStore{} - if filer.db, err = leveldb.OpenFile(dir, nil); err != nil { - return - } - return -} - -func (filer *EmbeddedStore) InsertEntry(entry *filer2.Entry) (err error) { - return nil -} - -func (filer *EmbeddedStore) UpdateEntry(entry *filer2.Entry) (err error) { - return nil -} - -func (filer *EmbeddedStore) FindEntry(fullpath filer2.FullPath) (found bool, entry *filer2.Entry, err error) { - return false, nil, nil -} - -func (filer *EmbeddedStore) DeleteEntry(fullpath filer2.FullPath) (entry *filer2.Entry, err error) { - return nil, nil -} - -func (filer *EmbeddedStore) ListDirectoryEntries(fullpath filer2.FullPath, startFileName string, inclusive bool, limit int) (entries []*filer2.Entry, err error) { - return nil, nil -} diff --git a/weed/filer2/entry.go b/weed/filer2/entry.go index 53d069ed1..f741d4bb3 100644 --- a/weed/filer2/entry.go +++ b/weed/filer2/entry.go @@ -28,15 +28,14 @@ type Entry struct { Chunks []*filer_pb.FileChunk `json:"chunks,omitempty"` } -func (entry Entry) Size() uint64 { +func (entry *Entry) Size() uint64 { return TotalSize(entry.Chunks) } -func (entry Entry) Timestamp() time.Time { +func (entry *Entry) Timestamp() time.Time { if entry.IsDirectory() { return entry.Crtime } else { return entry.Mtime } } - diff --git a/weed/filer2/entry_codec.go b/weed/filer2/entry_codec.go index 7d2b2da37..7fafea3e3 100644 --- a/weed/filer2/entry_codec.go +++ b/weed/filer2/entry_codec.go @@ -9,7 +9,7 @@ import ( "fmt" ) -func (entry Entry) EncodeAttributesAndChunks() ([]byte, error) { +func (entry *Entry) EncodeAttributesAndChunks() ([]byte, error) { message := &filer_pb.Entry{ Attributes: &filer_pb.FuseAttributes{ Crtime: entry.Attr.Crtime.Unix(), @@ -23,7 +23,7 @@ func (entry Entry) EncodeAttributesAndChunks() ([]byte, error) { return proto.Marshal(message) } -func (entry Entry) DecodeAttributesAndChunks(blob []byte) (error) { +func (entry *Entry) DecodeAttributesAndChunks(blob []byte) (error) { message := &filer_pb.Entry{} diff --git a/weed/filer2/filer.go b/weed/filer2/filer.go index 6b83db0e6..35d69b32e 100644 --- a/weed/filer2/filer.go +++ b/weed/filer2/filer.go @@ -50,11 +50,7 @@ func (f *Filer) CreateEntry(entry *Entry) (error) { // not found, check the store directly if dirEntry == nil { glog.V(4).Infof("find uncached directory: %s", dirPath) - var dirFindErr error - _, dirEntry, dirFindErr = f.FindEntry(FullPath(dirPath)) - if dirFindErr != nil { - return fmt.Errorf("findDirectory %s: %v", dirPath, dirFindErr) - } + dirEntry, _ = f.FindEntry(FullPath(dirPath)) } else { glog.V(4).Infof("found cached directory: %s", dirPath) } @@ -116,13 +112,13 @@ func (f *Filer) UpdateEntry(entry *Entry) (err error) { return f.store.UpdateEntry(entry) } -func (f *Filer) FindEntry(p FullPath) (found bool, entry *Entry, err error) { +func (f *Filer) FindEntry(p FullPath) (entry *Entry, err error) { return f.store.FindEntry(p) } func (f *Filer) DeleteEntry(p FullPath) (fileEntry *Entry, err error) { - found, entry, err := f.FindEntry(p) - if err != nil || !found { + entry, err := f.FindEntry(p) + if err != nil { return nil, err } if entry.IsDirectory() { diff --git a/weed/filer2/filerstore.go b/weed/filer2/filerstore.go index e90ec15ed..b9f7d1198 100644 --- a/weed/filer2/filerstore.go +++ b/weed/filer2/filerstore.go @@ -1,11 +1,16 @@ package filer2 -import "errors" +import ( + "errors" + "github.com/spf13/viper" +) type FilerStore interface { + GetName() string + Initialize(viper *viper.Viper) (error) InsertEntry(*Entry) (error) UpdateEntry(*Entry) (err error) - FindEntry(FullPath) (found bool, entry *Entry, err error) + FindEntry(FullPath) (entry *Entry, err error) DeleteEntry(FullPath) (fileEntry *Entry, err error) ListDirectoryEntries(dirPath FullPath, startFileName string, inclusive bool, limit int) ([]*Entry, error) } diff --git a/weed/filer2/leveldb/leveldb_store.go b/weed/filer2/leveldb/leveldb_store.go new file mode 100644 index 000000000..faf361f15 --- /dev/null +++ b/weed/filer2/leveldb/leveldb_store.go @@ -0,0 +1,171 @@ +package leveldb + +import ( + "fmt" + "bytes" + + "github.com/syndtr/goleveldb/leveldb" + "github.com/chrislusf/seaweedfs/weed/filer2" + leveldb_util "github.com/syndtr/goleveldb/leveldb/util" + "github.com/chrislusf/seaweedfs/weed/glog" + "github.com/spf13/viper" + weed_util "github.com/chrislusf/seaweedfs/weed/util" +) + +const ( + DIR_FILE_SEPARATOR = byte(0x00) +) + +func init() { + filer2.Stores = append(filer2.Stores, &LevelDBStore{}) +} + +type LevelDBStore struct { + db *leveldb.DB +} + +func (filer *LevelDBStore) GetName() string { + return "leveldb" +} + +func (filer *LevelDBStore) Initialize(viper *viper.Viper) (err error) { + dir := viper.GetString("dir") + return filer.initialize(dir) +} + +func (filer *LevelDBStore) initialize(dir string) (err error) { + if err := weed_util.TestFolderWritable(dir); err != nil { + return fmt.Errorf("Check Level Folder %s Writable: %s", dir, err) + } + + if filer.db, err = leveldb.OpenFile(dir, nil); err != nil { + return + } + return +} + +func (store *LevelDBStore) InsertEntry(entry *filer2.Entry) (err error) { + key := genKey(entry.DirAndName()) + + value, err := entry.EncodeAttributesAndChunks() + if err != nil { + return fmt.Errorf("encoding %s %+v: %v", entry.FullPath, entry.Attr, err) + } + + err = store.db.Put(key, value, nil) + + if err != nil { + return fmt.Errorf("persisting %s : %v", entry.FullPath, err) + } + + // println("saved", entry.FullPath, "chunks", len(entry.Chunks)) + + return nil +} + +func (store *LevelDBStore) UpdateEntry(entry *filer2.Entry) (err error) { + + return store.InsertEntry(entry) +} + +func (store *LevelDBStore) FindEntry(fullpath filer2.FullPath) (entry *filer2.Entry, err error) { + key := genKey(fullpath.DirAndName()) + + data, err := store.db.Get(key, nil) + + if err == leveldb.ErrNotFound { + return nil, filer2.ErrNotFound + } + if err != nil { + return nil, fmt.Errorf("get %s : %v", entry.FullPath, err) + } + + entry = &filer2.Entry{ + FullPath: fullpath, + } + err = entry.DecodeAttributesAndChunks(data) + if err != nil { + return entry, fmt.Errorf("decode %s : %v", entry.FullPath, err) + } + + // println("read", entry.FullPath, "chunks", len(entry.Chunks), "data", len(data), string(data)) + + return entry, nil +} + +func (store *LevelDBStore) DeleteEntry(fullpath filer2.FullPath) (entry *filer2.Entry, err error) { + key := genKey(fullpath.DirAndName()) + + entry, _ = store.FindEntry(fullpath) + + err = store.db.Delete(key, nil) + if err != nil { + return entry, fmt.Errorf("delete %s : %v", entry.FullPath, err) + } + + return entry, nil +} + +func (store *LevelDBStore) ListDirectoryEntries(fullpath filer2.FullPath, startFileName string, inclusive bool, + limit int) (entries []*filer2.Entry, err error) { + + directoryPrefix := genDirectoryKeyPrefix(fullpath, "") + + iter := store.db.NewIterator(&leveldb_util.Range{Start: genDirectoryKeyPrefix(fullpath, startFileName)}, nil) + for iter.Next() { + key := iter.Key() + if !bytes.HasPrefix(key, directoryPrefix) { + break + } + fileName := getNameFromKey(key) + if fileName == "" { + continue + } + if fileName == startFileName && !inclusive { + continue + } + limit-- + if limit < 0 { + break + } + entry := &filer2.Entry{ + FullPath: filer2.NewFullPath(string(fullpath), fileName), + } + if decodeErr := entry.DecodeAttributesAndChunks(iter.Value()); decodeErr != nil { + err = decodeErr + glog.V(0).Infof("list %s : %v", entry.FullPath, err) + break + } + entries = append(entries, entry) + } + iter.Release() + + return entries, err +} + +func genKey(dirPath, fileName string) (key []byte) { + key = []byte(dirPath) + key = append(key, DIR_FILE_SEPARATOR) + key = append(key, []byte(fileName)...) + return key +} + +func genDirectoryKeyPrefix(fullpath filer2.FullPath, startFileName string) (keyPrefix []byte) { + keyPrefix = []byte(string(fullpath)) + keyPrefix = append(keyPrefix, DIR_FILE_SEPARATOR) + if len(startFileName) > 0 { + keyPrefix = append(keyPrefix, []byte(startFileName)...) + } + return keyPrefix +} + +func getNameFromKey(key []byte) (string) { + + sepIndex := len(key) - 1 + for sepIndex >= 0 && key[sepIndex] != DIR_FILE_SEPARATOR { + sepIndex-- + } + + return string(key[sepIndex+1:]) + +} diff --git a/weed/filer2/leveldb/leveldb_store_test.go b/weed/filer2/leveldb/leveldb_store_test.go new file mode 100644 index 000000000..87f77d138 --- /dev/null +++ b/weed/filer2/leveldb/leveldb_store_test.go @@ -0,0 +1,61 @@ +package leveldb + +import ( + "testing" + "github.com/chrislusf/seaweedfs/weed/filer2" + "io/ioutil" + "os" +) + +func TestCreateAndFind(t *testing.T) { + filer := filer2.NewFiler("") + dir, _ := ioutil.TempDir("", "seaweedfs_filer_test") + defer os.RemoveAll(dir) + store := &LevelDBStore{} + store.initialize(dir) + filer.SetStore(store) + filer.DisableDirectoryCache() + + fullpath := filer2.FullPath("/home/chris/this/is/one/file1.jpg") + + entry1 := &filer2.Entry{ + FullPath: fullpath, + Attr: filer2.Attr{ + Mode: 0440, + Uid: 1234, + Gid: 5678, + }, + } + + if err := filer.CreateEntry(entry1); err != nil { + t.Errorf("create entry %v: %v", entry1.FullPath, err) + return + } + + entry, err := filer.FindEntry(fullpath) + + if err != nil { + t.Errorf("find entry: %v", err) + return + } + + if entry.FullPath != entry1.FullPath { + t.Errorf("find wrong entry: %v", entry.FullPath) + return + } + + // checking one upper directory + entries, _ := filer.ListDirectoryEntries(filer2.FullPath("/home/chris/this/is/one"), "", false, 100) + if len(entries) != 1 { + t.Errorf("list entries count: %v", len(entries)) + return + } + + // checking one upper directory + entries, _ = filer.ListDirectoryEntries(filer2.FullPath("/"), "", false, 100) + if len(entries) != 1 { + t.Errorf("list entries count: %v", len(entries)) + return + } + +} diff --git a/weed/filer2/memdb/memdb_store.go b/weed/filer2/memdb/memdb_store.go index 34757b00f..7d26f3dc7 100644 --- a/weed/filer2/memdb/memdb_store.go +++ b/weed/filer2/memdb/memdb_store.go @@ -6,8 +6,13 @@ import ( "strings" "fmt" "time" + "github.com/spf13/viper" ) +func init() { + filer2.Stores = append(filer2.Stores, &MemDbStore{}) +} + type MemDbStore struct { tree *btree.BTree } @@ -20,10 +25,13 @@ func (a Entry) Less(b btree.Item) bool { return strings.Compare(string(a.FullPath), string(b.(Entry).FullPath)) < 0 } -func NewMemDbStore() (filer *MemDbStore) { - filer = &MemDbStore{} +func (filer *MemDbStore) GetName() string { + return "memory" +} + +func (filer *MemDbStore) Initialize(viper *viper.Viper) (err error) { filer.tree = btree.New(8) - return + return nil } func (filer *MemDbStore) InsertEntry(entry *filer2.Entry) (err error) { @@ -34,22 +42,21 @@ func (filer *MemDbStore) InsertEntry(entry *filer2.Entry) (err error) { } func (filer *MemDbStore) UpdateEntry(entry *filer2.Entry) (err error) { - found, _, err := filer.FindEntry(entry.FullPath) - if !found { - return fmt.Errorf("No such file: %s", entry.FullPath) + if _, err = filer.FindEntry(entry.FullPath); err != nil { + return fmt.Errorf("no such file %s : %v", entry.FullPath, err) } entry.Mtime = time.Now() filer.tree.ReplaceOrInsert(Entry{entry}) return nil } -func (filer *MemDbStore) FindEntry(fullpath filer2.FullPath) (found bool, entry *filer2.Entry, err error) { +func (filer *MemDbStore) FindEntry(fullpath filer2.FullPath) (entry *filer2.Entry, err error) { item := filer.tree.Get(Entry{&filer2.Entry{FullPath: fullpath}}) if item == nil { - return false, nil, nil + return nil, nil } entry = item.(Entry).Entry - return true, entry, nil + return entry, nil } func (filer *MemDbStore) DeleteEntry(fullpath filer2.FullPath) (entry *filer2.Entry, err error) { diff --git a/weed/filer2/memdb/memdb_store_test.go b/weed/filer2/memdb/memdb_store_test.go index ab93a7665..b8d41bf8e 100644 --- a/weed/filer2/memdb/memdb_store_test.go +++ b/weed/filer2/memdb/memdb_store_test.go @@ -7,7 +7,9 @@ import ( func TestCreateAndFind(t *testing.T) { filer := filer2.NewFiler("") - filer.SetStore(NewMemDbStore()) + store := &MemDbStore{} + store.Initialize(nil) + filer.SetStore(store) filer.DisableDirectoryCache() fullpath := filer2.FullPath("/home/chris/this/is/one/file1.jpg") @@ -47,7 +49,9 @@ func TestCreateAndFind(t *testing.T) { func TestCreateFileAndList(t *testing.T) { filer := filer2.NewFiler("") - filer.SetStore(NewMemDbStore()) + store := &MemDbStore{} + store.Initialize(nil) + filer.SetStore(store) filer.DisableDirectoryCache() entry1 := &filer2.Entry{ diff --git a/weed/server/filer_grpc_server.go b/weed/server/filer_grpc_server.go index 0d1d7ae22..6aa8d50ff 100644 --- a/weed/server/filer_grpc_server.go +++ b/weed/server/filer_grpc_server.go @@ -14,12 +14,9 @@ import ( func (fs *FilerServer) LookupDirectoryEntry(ctx context.Context, req *filer_pb.LookupDirectoryEntryRequest) (*filer_pb.LookupDirectoryEntryResponse, error) { - found, entry, err := fs.filer.FindEntry(filer2.FullPath(filepath.Join(req.Directory, req.Name))) + entry, err := fs.filer.FindEntry(filer2.FullPath(filepath.Join(req.Directory, req.Name))) if err != nil { - return nil, err - } - if !found { - return nil, fmt.Errorf("%s not found under %s", req.Name, req.Directory) + return nil, fmt.Errorf("%s not found under %s: %v", req.Name, req.Directory, err) } return &filer_pb.LookupDirectoryEntryResponse{ @@ -65,13 +62,10 @@ func (fs *FilerServer) GetEntryAttributes(ctx context.Context, req *filer_pb.Get fullpath := filer2.NewFullPath(req.ParentDir, req.Name) - found, entry, err := fs.filer.FindEntry(fullpath) + entry, err := fs.filer.FindEntry(fullpath) if err != nil { - return nil, err - } - if !found { attributes.FileSize = 0 - return nil, fmt.Errorf("file %s not found", fullpath) + return nil, fmt.Errorf("FindEntry %s: %v", fullpath, err) } attributes.FileSize = entry.Size() @@ -138,12 +132,9 @@ func (fs *FilerServer) CreateEntry(ctx context.Context, req *filer_pb.CreateEntr func (fs *FilerServer) UpdateEntry(ctx context.Context, req *filer_pb.UpdateEntryRequest) (*filer_pb.UpdateEntryResponse, error) { fullpath := filepath.Join(req.Directory, req.Entry.Name) - found, entry, err := fs.filer.FindEntry(filer2.FullPath(fullpath)) + entry, err := fs.filer.FindEntry(filer2.FullPath(fullpath)) if err != nil { - return &filer_pb.UpdateEntryResponse{}, err - } - if !found { - return &filer_pb.UpdateEntryResponse{}, fmt.Errorf("file not found: %s", fullpath) + return &filer_pb.UpdateEntryResponse{}, fmt.Errorf("not found %s: %v", fullpath, err) } // remove old chunks if not included in the new ones diff --git a/weed/server/filer_server.go b/weed/server/filer_server.go index 8eeced9cb..8687edf6d 100644 --- a/weed/server/filer_server.go +++ b/weed/server/filer_server.go @@ -1,10 +1,8 @@ package weed_server import ( - "encoding/json" "math/rand" "net/http" - "os" "strconv" "sync" "time" @@ -16,7 +14,8 @@ import ( "github.com/chrislusf/seaweedfs/weed/storage" "github.com/chrislusf/seaweedfs/weed/util" "github.com/chrislusf/seaweedfs/weed/filer2" - "github.com/chrislusf/seaweedfs/weed/filer2/memdb" + _ "github.com/chrislusf/seaweedfs/weed/filer2/memdb" + _ "github.com/chrislusf/seaweedfs/weed/filer2/leveldb" ) type filerConf struct { @@ -25,21 +24,6 @@ type filerConf struct { PostgresConf *postgres_store.PostgresConf `json:"postgres"` } -func parseConfFile(confPath string) (*filerConf, error) { - var setting filerConf - configFile, err := os.Open(confPath) - defer configFile.Close() - if err != nil { - return nil, err - } - - jsonParser := json.NewDecoder(configFile) - if err = jsonParser.Decode(&setting); err != nil { - return nil, err - } - return &setting, nil -} - type FilerServer struct { port string master string @@ -54,13 +38,10 @@ type FilerServer struct { masterNodes *storage.MasterNodes } -func NewFilerServer(defaultMux, readonlyMux *http.ServeMux, ip string, port int, master string, dir string, collection string, +func NewFilerServer(defaultMux, readonlyMux *http.ServeMux, ip string, port int, master string, collection string, replication string, redirectOnRead bool, disableDirListing bool, - confFile string, maxMB int, secret string, - cassandra_server string, cassandra_keyspace string, - redis_server string, redis_password string, redis_database int, ) (fs *FilerServer, err error) { fs = &FilerServer{ master: master, @@ -71,42 +52,9 @@ func NewFilerServer(defaultMux, readonlyMux *http.ServeMux, ip string, port int, maxMB: maxMB, port: ip + ":" + strconv.Itoa(port), } - - var setting *filerConf - if confFile != "" { - setting, err = parseConfFile(confFile) - if err != nil { - return nil, err - } - } else { - setting = new(filerConf) - } - - if setting.MysqlConf != nil && len(setting.MysqlConf) != 0 { - // mysql_store := mysql_store.NewMysqlStore(setting.MysqlConf, setting.IsSharding, setting.ShardCount) - // fs.filer = flat_namespace.NewFlatNamespaceFiler(master, mysql_store) - } else if setting.PostgresConf != nil { - // fs.filer = postgres_store.NewPostgresStore(master, *setting.PostgresConf) - } else if cassandra_server != "" { - // cassandra_store, err := cassandra_store.NewCassandraStore(cassandra_keyspace, cassandra_server) - // if err != nil { - // glog.Fatalf("Can not connect to cassandra server %s with keyspace %s: %v", cassandra_server, cassandra_keyspace, err) - // } - // fs.filer = flat_namespace.NewFlatNamespaceFiler(master, cassandra_store) - } else if redis_server != "" { - // redis_store := redis_store.NewRedisStore(redis_server, redis_password, redis_database) - // fs.filer = flat_namespace.NewFlatNamespaceFiler(master, redis_store) - } else { - /* - if fs.filer, err = embedded_filer.NewFilerEmbedded(master, dir); err != nil { - glog.Fatalf("Can not start filer in dir %s : %v", dir, err) - return - } - */ - } - fs.filer = filer2.NewFiler(master) - fs.filer.SetStore(memdb.NewMemDbStore()) + + fs.filer.LoadConfiguration() defaultMux.HandleFunc("/admin/register", fs.registerHandler) defaultMux.HandleFunc("/", fs.filerHandler) diff --git a/weed/server/filer_server_handlers_admin.go b/weed/server/filer_server_handlers_admin.go index 01b18a76d..f3cf2754f 100644 --- a/weed/server/filer_server_handlers_admin.go +++ b/weed/server/filer_server_handlers_admin.go @@ -49,7 +49,7 @@ func (fs *FilerServer) registerHandler(w http.ResponseWriter, r *http.Request) { glog.V(2).Infof("register %s to %s parse fileSize %s", fileId, path, r.FormValue("fileSize")) err = fs.filer.CreateEntry(entry) if err != nil { - glog.V(4).Infof("register %s to %s error: %v", fileId, path, err) + glog.V(0).Infof("register %s to %s error: %v", fileId, path, err) writeJsonError(w, r, http.StatusInternalServerError, err) } else { w.WriteHeader(http.StatusOK) diff --git a/weed/server/filer_server_handlers_read.go b/weed/server/filer_server_handlers_read.go index 7c479e18b..e451a81de 100644 --- a/weed/server/filer_server_handlers_read.go +++ b/weed/server/filer_server_handlers_read.go @@ -79,9 +79,9 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request, path = path[:len(path)-1] } - found, entry, err := fs.filer.FindEntry(filer2.FullPath(path)) - if !found || err != nil { - glog.V(3).Infof("Not found %s: %v", path, err) + entry, err := fs.filer.FindEntry(filer2.FullPath(path)) + if err != nil { + glog.V(1).Infof("Not found %s: %v", path, err) w.WriteHeader(http.StatusNotFound) return } @@ -96,7 +96,7 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request, } if len(entry.Chunks) == 0 { - glog.V(3).Infof("Empty %s: %v", path) + glog.V(1).Infof("no file chunks for %s, attr=%+v", path, entry.Attr) w.WriteHeader(http.StatusNoContent) return } diff --git a/weed/server/filer_server_handlers_write.go b/weed/server/filer_server_handlers_write.go index c6e735cf5..a24995956 100644 --- a/weed/server/filer_server_handlers_write.go +++ b/weed/server/filer_server_handlers_write.go @@ -76,20 +76,17 @@ func makeFormData(filename, mimeType string, content io.Reader) (formData io.Rea } func (fs *FilerServer) queryFileInfoByPath(w http.ResponseWriter, r *http.Request, path string) (fileId, urlLocation string, err error) { - var found bool var entry *filer2.Entry - if found, entry, err = fs.filer.FindEntry(filer2.FullPath(path)); err != nil { + if entry, err = fs.filer.FindEntry(filer2.FullPath(path)); err != nil { glog.V(0).Infoln("failing to find path in filer store", path, err.Error()) writeJsonError(w, r, http.StatusInternalServerError, err) - } else if found { + } else { fileId = entry.Chunks[0].FileId urlLocation, err = operation.LookupFileId(fs.getMasterNode(), fileId) if err != nil { glog.V(1).Infoln("operation LookupFileId %s failed, err is %s", fileId, err.Error()) w.WriteHeader(http.StatusNotFound) } - } else { - w.WriteHeader(http.StatusNotFound) } return } @@ -319,7 +316,7 @@ func (fs *FilerServer) PostHandler(w http.ResponseWriter, r *http.Request) { // also delete the old fid unless PUT operation if r.Method != "PUT" { - if found, entry, err := fs.filer.FindEntry(filer2.FullPath(path)); err == nil && found { + if entry, err := fs.filer.FindEntry(filer2.FullPath(path)); err == nil { oldFid := entry.Chunks[0].FileId operation.DeleteFile(fs.getMasterNode(), oldFid, fs.jwt(oldFid)) } else if err != nil && err != filer.ErrNotFound { @@ -485,7 +482,7 @@ func (fs *FilerServer) doAutoChunk(w http.ResponseWriter, r *http.Request, conte path := r.URL.Path // also delete the old fid unless PUT operation if r.Method != "PUT" { - if found, entry, err := fs.filer.FindEntry(filer2.FullPath(path)); found && err == nil { + if entry, err := fs.filer.FindEntry(filer2.FullPath(path)); err == nil { for _, chunk := range entry.Chunks { oldFid := chunk.FileId operation.DeleteFile(fs.getMasterNode(), oldFid, fs.jwt(oldFid)) From 828e4a5ace44eecd53164d08ce7fe81a4ac348bf Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sat, 26 May 2018 04:13:50 -0700 Subject: [PATCH 40/83] fix test error --- weed/filer2/memdb/memdb_store_test.go | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/weed/filer2/memdb/memdb_store_test.go b/weed/filer2/memdb/memdb_store_test.go index b8d41bf8e..a8631f969 100644 --- a/weed/filer2/memdb/memdb_store_test.go +++ b/weed/filer2/memdb/memdb_store_test.go @@ -28,18 +28,13 @@ func TestCreateAndFind(t *testing.T) { return } - found, entry, err := filer.FindEntry(fullpath) + entry, err := filer.FindEntry(fullpath) if err != nil { t.Errorf("find entry: %v", err) return } - if !found { - t.Errorf("Failed to find newly created file") - return - } - if entry.FullPath != entry1.FullPath { t.Errorf("find wrong entry: %v", entry.FullPath) return From c1e353491a1ff048705e75b40427e92623809fe3 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sat, 26 May 2018 04:50:55 -0700 Subject: [PATCH 41/83] refactoring variable names --- weed/filer2/leveldb/leveldb_store.go | 10 +++---- weed/filer2/memdb/memdb_store.go | 40 ++++++++++++++-------------- 2 files changed, 25 insertions(+), 25 deletions(-) diff --git a/weed/filer2/leveldb/leveldb_store.go b/weed/filer2/leveldb/leveldb_store.go index faf361f15..00fb44d36 100644 --- a/weed/filer2/leveldb/leveldb_store.go +++ b/weed/filer2/leveldb/leveldb_store.go @@ -24,21 +24,21 @@ type LevelDBStore struct { db *leveldb.DB } -func (filer *LevelDBStore) GetName() string { +func (store *LevelDBStore) GetName() string { return "leveldb" } -func (filer *LevelDBStore) Initialize(viper *viper.Viper) (err error) { +func (store *LevelDBStore) Initialize(viper *viper.Viper) (err error) { dir := viper.GetString("dir") - return filer.initialize(dir) + return store.initialize(dir) } -func (filer *LevelDBStore) initialize(dir string) (err error) { +func (store *LevelDBStore) initialize(dir string) (err error) { if err := weed_util.TestFolderWritable(dir); err != nil { return fmt.Errorf("Check Level Folder %s Writable: %s", dir, err) } - if filer.db, err = leveldb.OpenFile(dir, nil); err != nil { + if store.db, err = leveldb.OpenFile(dir, nil); err != nil { return } return diff --git a/weed/filer2/memdb/memdb_store.go b/weed/filer2/memdb/memdb_store.go index 7d26f3dc7..e5b06ce47 100644 --- a/weed/filer2/memdb/memdb_store.go +++ b/weed/filer2/memdb/memdb_store.go @@ -17,70 +17,70 @@ type MemDbStore struct { tree *btree.BTree } -type Entry struct { +type entryItem struct { *filer2.Entry } -func (a Entry) Less(b btree.Item) bool { - return strings.Compare(string(a.FullPath), string(b.(Entry).FullPath)) < 0 +func (a entryItem) Less(b btree.Item) bool { + return strings.Compare(string(a.FullPath), string(b.(entryItem).FullPath)) < 0 } -func (filer *MemDbStore) GetName() string { +func (store *MemDbStore) GetName() string { return "memory" } -func (filer *MemDbStore) Initialize(viper *viper.Viper) (err error) { - filer.tree = btree.New(8) +func (store *MemDbStore) Initialize(viper *viper.Viper) (err error) { + store.tree = btree.New(8) return nil } -func (filer *MemDbStore) InsertEntry(entry *filer2.Entry) (err error) { +func (store *MemDbStore) InsertEntry(entry *filer2.Entry) (err error) { // println("inserting", entry.FullPath) entry.Crtime = time.Now() - filer.tree.ReplaceOrInsert(Entry{entry}) + store.tree.ReplaceOrInsert(entryItem{entry}) return nil } -func (filer *MemDbStore) UpdateEntry(entry *filer2.Entry) (err error) { - if _, err = filer.FindEntry(entry.FullPath); err != nil { +func (store *MemDbStore) UpdateEntry(entry *filer2.Entry) (err error) { + if _, err = store.FindEntry(entry.FullPath); err != nil { return fmt.Errorf("no such file %s : %v", entry.FullPath, err) } entry.Mtime = time.Now() - filer.tree.ReplaceOrInsert(Entry{entry}) + store.tree.ReplaceOrInsert(entryItem{entry}) return nil } -func (filer *MemDbStore) FindEntry(fullpath filer2.FullPath) (entry *filer2.Entry, err error) { - item := filer.tree.Get(Entry{&filer2.Entry{FullPath: fullpath}}) +func (store *MemDbStore) FindEntry(fullpath filer2.FullPath) (entry *filer2.Entry, err error) { + item := store.tree.Get(entryItem{&filer2.Entry{FullPath: fullpath}}) if item == nil { return nil, nil } - entry = item.(Entry).Entry + entry = item.(entryItem).Entry return entry, nil } -func (filer *MemDbStore) DeleteEntry(fullpath filer2.FullPath) (entry *filer2.Entry, err error) { - item := filer.tree.Delete(Entry{&filer2.Entry{FullPath: fullpath}}) +func (store *MemDbStore) DeleteEntry(fullpath filer2.FullPath) (entry *filer2.Entry, err error) { + item := store.tree.Delete(entryItem{&filer2.Entry{FullPath: fullpath}}) if item == nil { return nil, nil } - entry = item.(Entry).Entry + entry = item.(entryItem).Entry return entry, nil } -func (filer *MemDbStore) ListDirectoryEntries(fullpath filer2.FullPath, startFileName string, inclusive bool, limit int) (entries []*filer2.Entry, err error) { +func (store *MemDbStore) ListDirectoryEntries(fullpath filer2.FullPath, startFileName string, inclusive bool, limit int) (entries []*filer2.Entry, err error) { startFrom := string(fullpath) if startFileName != "" { startFrom = startFrom + "/" + startFileName } - filer.tree.AscendGreaterOrEqual(Entry{&filer2.Entry{FullPath: filer2.FullPath(startFrom)}}, + store.tree.AscendGreaterOrEqual(entryItem{&filer2.Entry{FullPath: filer2.FullPath(startFrom)}}, func(item btree.Item) bool { if limit <= 0 { return false } - entry := item.(Entry).Entry + entry := item.(entryItem).Entry // println("checking", entry.FullPath) if entry.FullPath == fullpath { From 68bcaff14d6630c4549474125f631d9093502828 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sat, 26 May 2018 05:32:15 -0700 Subject: [PATCH 42/83] mysql can compile, not yet tested! --- .../filer2/abstract_sql/abstract_sql_store.go | 121 ++++++++++++++++++ weed/filer2/configuration.go | 15 ++- weed/filer2/memdb/memdb_store.go | 3 - weed/filer2/mysql/mysql_store.go | 54 ++++++++ weed/server/filer_server.go | 11 +- 5 files changed, 188 insertions(+), 16 deletions(-) create mode 100644 weed/filer2/abstract_sql/abstract_sql_store.go create mode 100644 weed/filer2/mysql/mysql_store.go diff --git a/weed/filer2/abstract_sql/abstract_sql_store.go b/weed/filer2/abstract_sql/abstract_sql_store.go new file mode 100644 index 000000000..252a3c549 --- /dev/null +++ b/weed/filer2/abstract_sql/abstract_sql_store.go @@ -0,0 +1,121 @@ +package abstract_sql + +import ( + "fmt" + "database/sql" + + "github.com/chrislusf/seaweedfs/weed/filer2" + "github.com/chrislusf/seaweedfs/weed/glog" +) + +type AbstractSqlStore struct { + DB *sql.DB +} + +func (store *AbstractSqlStore) InsertEntry(entry *filer2.Entry) (err error) { + + dir, name := entry.FullPath.DirAndName() + meta, err := entry.EncodeAttributesAndChunks() + if err != nil { + return fmt.Errorf("mysql encode %s: %s", entry.FullPath, err) + } + + res, err := store.DB.Exec("INSERT INTO seaweedfs (directory,name,meta) VALUES(?,?,?)", dir, name, meta) + if err != nil { + return fmt.Errorf("mysql insert %s: %s", entry.FullPath, err) + } + + _, err = res.RowsAffected() + if err != nil { + return fmt.Errorf("mysql insert %s but no rows affected: %s", entry.FullPath, err) + } + return nil +} + +func (store *AbstractSqlStore) UpdateEntry(entry *filer2.Entry) (err error) { + + dir, name := entry.FullPath.DirAndName() + meta, err := entry.EncodeAttributesAndChunks() + if err != nil { + return fmt.Errorf("mysql encode %s: %s", entry.FullPath, err) + } + + res, err := store.DB.Exec("UPDATE seaweedfs SET meta=? WHERE directory=? and name=?", dir, name, meta) + if err != nil { + return fmt.Errorf("mysql update %s: %s", entry.FullPath, err) + } + + _, err = res.RowsAffected() + if err != nil { + return fmt.Errorf("mysql update %s but no rows affected: %s", entry.FullPath, err) + } + return nil +} + +func (store *AbstractSqlStore) FindEntry(fullpath filer2.FullPath) (*filer2.Entry, error) { + + dir, name := fullpath.DirAndName() + row := store.DB.QueryRow("SELECT meta FROM seaweedfs WHERE directory=? and name=?", dir, name) + var data []byte + if err := row.Scan(&data); err != nil { + return nil, fmt.Errorf("mysql read entry %s: %v", fullpath, err) + } + + entry := &filer2.Entry{ + FullPath: fullpath, + } + if err := entry.DecodeAttributesAndChunks(data); err != nil { + return entry, fmt.Errorf("mysql decode %s : %v", entry.FullPath, err) + } + + return entry, nil +} + +func (store *AbstractSqlStore) DeleteEntry(fullpath filer2.FullPath) (*filer2.Entry, error) { + + entry, _ := store.FindEntry(fullpath) + + dir, name := fullpath.DirAndName() + + res, err := store.DB.Exec("DELETE FROM seaweedfs WHERE directory=? and name=?", dir, name) + if err != nil { + return nil, fmt.Errorf("mysql delete %s: %s", fullpath, err) + } + + _, err = res.RowsAffected() + if err != nil { + return nil, fmt.Errorf("mysql delete %s but no rows affected: %s", fullpath, err) + } + + return entry, nil +} + +func (store *AbstractSqlStore) ListDirectoryEntries(fullpath filer2.FullPath, startFileName string, inclusive bool, limit int) (entries []*filer2.Entry, err error) { + + rows, err := store.DB.Query("SELECT NAME, meta FROM seaweedfs WHERE directory=? and name>?", fullpath, startFileName) + if err != nil { + return nil, fmt.Errorf("mysql list %s : %v", fullpath, err) + } + defer rows.Close() + + for rows.Next() { + var name string + var data []byte + if err = rows.Scan(&name, &data); err != nil { + glog.V(0).Infof("mysql scan %s : %v", fullpath, err) + return nil, fmt.Errorf("mysql scan %s: %v", fullpath, err) + } + + entry := &filer2.Entry{ + FullPath: fullpath, + } + if err = entry.DecodeAttributesAndChunks(data); err != nil { + glog.V(0).Infof("mysql scan decode %s : %v", entry.FullPath, err) + return nil, fmt.Errorf("mysql scan decode %s : %v", entry.FullPath, err) + } + + entries = append(entries, entry) + } + + return entries, nil +} diff --git a/weed/filer2/configuration.go b/weed/filer2/configuration.go index 6dabf10b4..caca0a49c 100644 --- a/weed/filer2/configuration.go +++ b/weed/filer2/configuration.go @@ -20,13 +20,20 @@ enabled = false dir = "." # directory to store level db files [mysql] +# also need to manually create seaweedfs table. +# CREATE TABLE IF NOT EXISTS seaweedfs ( +# directory VARCHAR(4096) NOT NULL DEFAULT "" COMMENT 'full path to parent directory', +# name VARCHAR(1024) NOT NULL DEFAULT "" COMMENT 'directory or file name', +# meta BLOB, +# PRIMARY KEY (directory, name), +# ) DEFAULT CHARSET=utf8; enabled = true -server = "192.168.1.1" -port = 8080 -username = "" +server = "localhost" +port = 3306 +username = "root" password = "" database = "" -connection_max_idle = 100 +connection_max_idle = 2 connection_max_open = 100 [postgres] diff --git a/weed/filer2/memdb/memdb_store.go b/weed/filer2/memdb/memdb_store.go index e5b06ce47..452e4970c 100644 --- a/weed/filer2/memdb/memdb_store.go +++ b/weed/filer2/memdb/memdb_store.go @@ -5,7 +5,6 @@ import ( "github.com/google/btree" "strings" "fmt" - "time" "github.com/spf13/viper" ) @@ -36,7 +35,6 @@ func (store *MemDbStore) Initialize(viper *viper.Viper) (err error) { func (store *MemDbStore) InsertEntry(entry *filer2.Entry) (err error) { // println("inserting", entry.FullPath) - entry.Crtime = time.Now() store.tree.ReplaceOrInsert(entryItem{entry}) return nil } @@ -45,7 +43,6 @@ func (store *MemDbStore) UpdateEntry(entry *filer2.Entry) (err error) { if _, err = store.FindEntry(entry.FullPath); err != nil { return fmt.Errorf("no such file %s : %v", entry.FullPath, err) } - entry.Mtime = time.Now() store.tree.ReplaceOrInsert(entryItem{entry}) return nil } diff --git a/weed/filer2/mysql/mysql_store.go b/weed/filer2/mysql/mysql_store.go new file mode 100644 index 000000000..4accc9a9f --- /dev/null +++ b/weed/filer2/mysql/mysql_store.go @@ -0,0 +1,54 @@ +package mysql + +import ( + "fmt" + "database/sql" + + "github.com/chrislusf/seaweedfs/weed/filer2" + "github.com/spf13/viper" + _ "github.com/go-sql-driver/mysql" + "github.com/chrislusf/seaweedfs/weed/filer2/abstract_sql" +) + +const ( + CONNECTION_URL_PATTERN = "%s:%s@tcp(%s:%d)/%s?charset=utf8" +) + +func init() { + filer2.Stores = append(filer2.Stores, &MysqlStore{}) +} + +type MysqlStore struct { + abstract_sql.AbstractSqlStore +} + +func (store *MysqlStore) GetName() string { + return "mysql" +} + +func (store *MysqlStore) Initialize(viper *viper.Viper) (err error) { + return store.initialize( + viper.GetString("username"), + viper.GetString("password"), + viper.GetString("hostname"), + viper.GetString("port"), + viper.GetString("database"), + viper.GetInt("connection_max_idle"), + viper.GetInt("connection_max_open"), + ) +} + +func (store *MysqlStore) initialize(user, password, hostname, port, database string, maxIdle, maxOpen int) (err error) { + sqlUrl := fmt.Sprintf(CONNECTION_URL_PATTERN, user, password, hostname, port, database) + var dbErr error + store.DB, dbErr = sql.Open("mysql", sqlUrl) + if dbErr != nil { + store.DB.Close() + store.DB = nil + return fmt.Errorf("can not connect to %s error:%v", sqlUrl, err) + } + + store.DB.SetMaxIdleConns(maxIdle) + store.DB.SetMaxOpenConns(maxOpen) + return nil +} diff --git a/weed/server/filer_server.go b/weed/server/filer_server.go index 8687edf6d..2e88252e9 100644 --- a/weed/server/filer_server.go +++ b/weed/server/filer_server.go @@ -7,23 +7,16 @@ import ( "sync" "time" - "github.com/chrislusf/seaweedfs/weed/filer/mysql_store" - "github.com/chrislusf/seaweedfs/weed/filer/postgres_store" "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/security" "github.com/chrislusf/seaweedfs/weed/storage" "github.com/chrislusf/seaweedfs/weed/util" "github.com/chrislusf/seaweedfs/weed/filer2" - _ "github.com/chrislusf/seaweedfs/weed/filer2/memdb" _ "github.com/chrislusf/seaweedfs/weed/filer2/leveldb" + _ "github.com/chrislusf/seaweedfs/weed/filer2/memdb" + _ "github.com/chrislusf/seaweedfs/weed/filer2/mysql" ) -type filerConf struct { - MysqlConf []mysql_store.MySqlConf `json:"mysql"` - mysql_store.ShardingConf - PostgresConf *postgres_store.PostgresConf `json:"postgres"` -} - type FilerServer struct { port string master string From 2da84ed33115a4f2908479c86442b1936ed1b616 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sat, 26 May 2018 13:35:56 -0700 Subject: [PATCH 43/83] working fine now but index length can be improved --- weed/filer2/abstract_sql/abstract_sql_store.go | 17 +++++++++++------ weed/filer2/configuration.go | 18 +++++++++++------- weed/filer2/mysql/mysql_store.go | 9 +++++++-- 3 files changed, 29 insertions(+), 15 deletions(-) diff --git a/weed/filer2/abstract_sql/abstract_sql_store.go b/weed/filer2/abstract_sql/abstract_sql_store.go index 252a3c549..dd037b6af 100644 --- a/weed/filer2/abstract_sql/abstract_sql_store.go +++ b/weed/filer2/abstract_sql/abstract_sql_store.go @@ -20,7 +20,7 @@ func (store *AbstractSqlStore) InsertEntry(entry *filer2.Entry) (err error) { return fmt.Errorf("mysql encode %s: %s", entry.FullPath, err) } - res, err := store.DB.Exec("INSERT INTO seaweedfs (directory,name,meta) VALUES(?,?,?)", dir, name, meta) + res, err := store.DB.Exec("INSERT INTO filemeta (directory,name,meta) VALUES(?,?,?)", dir, name, meta) if err != nil { return fmt.Errorf("mysql insert %s: %s", entry.FullPath, err) } @@ -40,7 +40,7 @@ func (store *AbstractSqlStore) UpdateEntry(entry *filer2.Entry) (err error) { return fmt.Errorf("mysql encode %s: %s", entry.FullPath, err) } - res, err := store.DB.Exec("UPDATE seaweedfs SET meta=? WHERE directory=? and name=?", dir, name, meta) + res, err := store.DB.Exec("UPDATE filemeta SET meta=? WHERE directory=? and name=?", dir, name, meta) if err != nil { return fmt.Errorf("mysql update %s: %s", entry.FullPath, err) } @@ -55,7 +55,7 @@ func (store *AbstractSqlStore) UpdateEntry(entry *filer2.Entry) (err error) { func (store *AbstractSqlStore) FindEntry(fullpath filer2.FullPath) (*filer2.Entry, error) { dir, name := fullpath.DirAndName() - row := store.DB.QueryRow("SELECT meta FROM seaweedfs WHERE directory=? and name=?", dir, name) + row := store.DB.QueryRow("SELECT meta FROM filemeta WHERE directory=? and name=?", dir, name) var data []byte if err := row.Scan(&data); err != nil { return nil, fmt.Errorf("mysql read entry %s: %v", fullpath, err) @@ -77,7 +77,7 @@ func (store *AbstractSqlStore) DeleteEntry(fullpath filer2.FullPath) (*filer2.En dir, name := fullpath.DirAndName() - res, err := store.DB.Exec("DELETE FROM seaweedfs WHERE directory=? and name=?", dir, name) + res, err := store.DB.Exec("DELETE FROM filemeta WHERE directory=? and name=?", dir, name) if err != nil { return nil, fmt.Errorf("mysql delete %s: %s", fullpath, err) } @@ -92,7 +92,12 @@ func (store *AbstractSqlStore) DeleteEntry(fullpath filer2.FullPath) (*filer2.En func (store *AbstractSqlStore) ListDirectoryEntries(fullpath filer2.FullPath, startFileName string, inclusive bool, limit int) (entries []*filer2.Entry, err error) { - rows, err := store.DB.Query("SELECT NAME, meta FROM seaweedfs WHERE directory=? and name>?", fullpath, startFileName) + sqlText := "SELECT NAME, meta FROM filemeta WHERE directory=? and name>? LIMIT ?" + if inclusive { + sqlText = "SELECT NAME, meta FROM filemeta WHERE directory=? and name>=? LIMIT ?" + } + + rows, err := store.DB.Query(sqlText, string(fullpath), startFileName, limit) if err != nil { return nil, fmt.Errorf("mysql list %s : %v", fullpath, err) } @@ -107,7 +112,7 @@ func (store *AbstractSqlStore) ListDirectoryEntries(fullpath filer2.FullPath, st } entry := &filer2.Entry{ - FullPath: fullpath, + FullPath: filer2.NewFullPath(string(fullpath), name), } if err = entry.DecodeAttributesAndChunks(data); err != nil { glog.V(0).Infof("mysql scan decode %s : %v", entry.FullPath, err) diff --git a/weed/filer2/configuration.go b/weed/filer2/configuration.go index caca0a49c..ddd06a568 100644 --- a/weed/filer2/configuration.go +++ b/weed/filer2/configuration.go @@ -11,28 +11,32 @@ const ( FILER_TOML_EXAMPLE = ` # A sample TOML config file for SeaweedFS filer store -# local in memory, mostly for testing purpose [memory] +# local in memory, mostly for testing purpose enabled = false [leveldb] +# local on disk, mostly for simple single-machine setup, fairly scalable enabled = false dir = "." # directory to store level db files [mysql] -# also need to manually create seaweedfs table. -# CREATE TABLE IF NOT EXISTS seaweedfs ( -# directory VARCHAR(4096) NOT NULL DEFAULT "" COMMENT 'full path to parent directory', -# name VARCHAR(1024) NOT NULL DEFAULT "" COMMENT 'directory or file name', +# multiple filers on shared storage, fairly scalable +# +# need to choose or create a database. +# need to manually create a table "filemeta". +# CREATE TABLE IF NOT EXISTS filemeta ( +# directory VARCHAR(512) NOT NULL DEFAULT "" COMMENT 'full path to parent directory', +# name VARCHAR(512) NOT NULL DEFAULT "" COMMENT 'directory or file name', # meta BLOB, -# PRIMARY KEY (directory, name), +# PRIMARY KEY (directory, name) # ) DEFAULT CHARSET=utf8; enabled = true server = "localhost" port = 3306 username = "root" password = "" -database = "" +database = "" # create or use an existing database, create the seaweedfs table. connection_max_idle = 2 connection_max_open = 100 diff --git a/weed/filer2/mysql/mysql_store.go b/weed/filer2/mysql/mysql_store.go index 4accc9a9f..16fd4311f 100644 --- a/weed/filer2/mysql/mysql_store.go +++ b/weed/filer2/mysql/mysql_store.go @@ -31,14 +31,14 @@ func (store *MysqlStore) Initialize(viper *viper.Viper) (err error) { viper.GetString("username"), viper.GetString("password"), viper.GetString("hostname"), - viper.GetString("port"), + viper.GetInt("port"), viper.GetString("database"), viper.GetInt("connection_max_idle"), viper.GetInt("connection_max_open"), ) } -func (store *MysqlStore) initialize(user, password, hostname, port, database string, maxIdle, maxOpen int) (err error) { +func (store *MysqlStore) initialize(user, password, hostname string, port int, database string, maxIdle, maxOpen int) (err error) { sqlUrl := fmt.Sprintf(CONNECTION_URL_PATTERN, user, password, hostname, port, database) var dbErr error store.DB, dbErr = sql.Open("mysql", sqlUrl) @@ -50,5 +50,10 @@ func (store *MysqlStore) initialize(user, password, hostname, port, database str store.DB.SetMaxIdleConns(maxIdle) store.DB.SetMaxOpenConns(maxOpen) + + if err = store.DB.Ping(); err != nil { + return fmt.Errorf("connect to %s error:%v", sqlUrl, err) + } + return nil } From 955eae3500ef64be7ff4cd33396253d0da333633 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sat, 26 May 2018 14:08:55 -0700 Subject: [PATCH 44/83] this works great and support long file names --- weed/filer2/abstract_sql/abstract_sql_store.go | 18 +++++++++++------- weed/filer2/abstract_sql/hashing.go | 13 +++++++++++++ weed/filer2/configuration.go | 9 ++++++--- 3 files changed, 30 insertions(+), 10 deletions(-) create mode 100644 weed/filer2/abstract_sql/hashing.go diff --git a/weed/filer2/abstract_sql/abstract_sql_store.go b/weed/filer2/abstract_sql/abstract_sql_store.go index dd037b6af..3d15c632d 100644 --- a/weed/filer2/abstract_sql/abstract_sql_store.go +++ b/weed/filer2/abstract_sql/abstract_sql_store.go @@ -20,7 +20,8 @@ func (store *AbstractSqlStore) InsertEntry(entry *filer2.Entry) (err error) { return fmt.Errorf("mysql encode %s: %s", entry.FullPath, err) } - res, err := store.DB.Exec("INSERT INTO filemeta (directory,name,meta) VALUES(?,?,?)", dir, name, meta) + res, err := store.DB.Exec("INSERT INTO filemeta (dirhash,name,directory,meta) VALUES(?,?,?,?)", + md5hash(dir), name, dir, meta) if err != nil { return fmt.Errorf("mysql insert %s: %s", entry.FullPath, err) } @@ -40,7 +41,8 @@ func (store *AbstractSqlStore) UpdateEntry(entry *filer2.Entry) (err error) { return fmt.Errorf("mysql encode %s: %s", entry.FullPath, err) } - res, err := store.DB.Exec("UPDATE filemeta SET meta=? WHERE directory=? and name=?", dir, name, meta) + res, err := store.DB.Exec("UPDATE filemeta SET meta=? WHERE dirhash=? AND name=? AND directory=?", + meta, md5hash(dir), name, dir) if err != nil { return fmt.Errorf("mysql update %s: %s", entry.FullPath, err) } @@ -55,7 +57,8 @@ func (store *AbstractSqlStore) UpdateEntry(entry *filer2.Entry) (err error) { func (store *AbstractSqlStore) FindEntry(fullpath filer2.FullPath) (*filer2.Entry, error) { dir, name := fullpath.DirAndName() - row := store.DB.QueryRow("SELECT meta FROM filemeta WHERE directory=? and name=?", dir, name) + row := store.DB.QueryRow("SELECT meta FROM filemeta WHERE dirhash=? AND name=? AND directory=?", + md5hash(dir), name, dir) var data []byte if err := row.Scan(&data); err != nil { return nil, fmt.Errorf("mysql read entry %s: %v", fullpath, err) @@ -77,7 +80,8 @@ func (store *AbstractSqlStore) DeleteEntry(fullpath filer2.FullPath) (*filer2.En dir, name := fullpath.DirAndName() - res, err := store.DB.Exec("DELETE FROM filemeta WHERE directory=? and name=?", dir, name) + res, err := store.DB.Exec("DELETE FROM filemeta WHERE dirhash=? AND name=? AND directory=?", + md5hash(dir), name, dir) if err != nil { return nil, fmt.Errorf("mysql delete %s: %s", fullpath, err) } @@ -92,12 +96,12 @@ func (store *AbstractSqlStore) DeleteEntry(fullpath filer2.FullPath) (*filer2.En func (store *AbstractSqlStore) ListDirectoryEntries(fullpath filer2.FullPath, startFileName string, inclusive bool, limit int) (entries []*filer2.Entry, err error) { - sqlText := "SELECT NAME, meta FROM filemeta WHERE directory=? and name>? LIMIT ?" + sqlText := "SELECT NAME, meta FROM filemeta WHERE dirhash=? AND name>? AND directory=? LIMIT ?" if inclusive { - sqlText = "SELECT NAME, meta FROM filemeta WHERE directory=? and name>=? LIMIT ?" + sqlText = "SELECT NAME, meta FROM filemeta WHERE dirhash=? AND name>=? AND directory=? LIMIT ?" } - rows, err := store.DB.Query(sqlText, string(fullpath), startFileName, limit) + rows, err := store.DB.Query(sqlText, md5hash(string(fullpath)), startFileName, string(fullpath), limit) if err != nil { return nil, fmt.Errorf("mysql list %s : %v", fullpath, err) } diff --git a/weed/filer2/abstract_sql/hashing.go b/weed/filer2/abstract_sql/hashing.go new file mode 100644 index 000000000..5e1390233 --- /dev/null +++ b/weed/filer2/abstract_sql/hashing.go @@ -0,0 +1,13 @@ +package abstract_sql + +import ( + "crypto/md5" + "io" +) + +// returns a 128 bit hash +func md5hash(dir string) []byte { + h := md5.New() + io.WriteString(h, dir) + return h.Sum(nil) +} diff --git a/weed/filer2/configuration.go b/weed/filer2/configuration.go index ddd06a568..910ddcbbc 100644 --- a/weed/filer2/configuration.go +++ b/weed/filer2/configuration.go @@ -25,12 +25,15 @@ dir = "." # directory to store level db files # # need to choose or create a database. # need to manually create a table "filemeta". +# # CREATE TABLE IF NOT EXISTS filemeta ( -# directory VARCHAR(512) NOT NULL DEFAULT "" COMMENT 'full path to parent directory', -# name VARCHAR(512) NOT NULL DEFAULT "" COMMENT 'directory or file name', +# dirhash BINARY(16) COMMENT 'MD5 hash value of directory field', +# name VARCHAR(1000) NOT NULL DEFAULT "" COMMENT 'directory or file name', +# directory VARCHAR(4096) NOT NULL DEFAULT "" COMMENT 'full path to parent directory', # meta BLOB, -# PRIMARY KEY (directory, name) +# PRIMARY KEY (dirhash, name) # ) DEFAULT CHARSET=utf8; +# enabled = true server = "localhost" port = 3306 From 87b3b84471c7041c29474840616cd3f15705eabe Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sat, 26 May 2018 21:24:03 -0700 Subject: [PATCH 45/83] simplifying dirhash to 64bit integer --- .../filer2/abstract_sql/abstract_sql_store.go | 10 ++++---- weed/filer2/abstract_sql/hashing.go | 25 ++++++++++++++++--- weed/filer2/configuration.go | 19 +++++++++----- 3 files changed, 40 insertions(+), 14 deletions(-) diff --git a/weed/filer2/abstract_sql/abstract_sql_store.go b/weed/filer2/abstract_sql/abstract_sql_store.go index 3d15c632d..900056dae 100644 --- a/weed/filer2/abstract_sql/abstract_sql_store.go +++ b/weed/filer2/abstract_sql/abstract_sql_store.go @@ -21,7 +21,7 @@ func (store *AbstractSqlStore) InsertEntry(entry *filer2.Entry) (err error) { } res, err := store.DB.Exec("INSERT INTO filemeta (dirhash,name,directory,meta) VALUES(?,?,?,?)", - md5hash(dir), name, dir, meta) + hashToLong(dir), name, dir, meta) if err != nil { return fmt.Errorf("mysql insert %s: %s", entry.FullPath, err) } @@ -42,7 +42,7 @@ func (store *AbstractSqlStore) UpdateEntry(entry *filer2.Entry) (err error) { } res, err := store.DB.Exec("UPDATE filemeta SET meta=? WHERE dirhash=? AND name=? AND directory=?", - meta, md5hash(dir), name, dir) + meta, hashToLong(dir), name, dir) if err != nil { return fmt.Errorf("mysql update %s: %s", entry.FullPath, err) } @@ -58,7 +58,7 @@ func (store *AbstractSqlStore) FindEntry(fullpath filer2.FullPath) (*filer2.Entr dir, name := fullpath.DirAndName() row := store.DB.QueryRow("SELECT meta FROM filemeta WHERE dirhash=? AND name=? AND directory=?", - md5hash(dir), name, dir) + hashToLong(dir), name, dir) var data []byte if err := row.Scan(&data); err != nil { return nil, fmt.Errorf("mysql read entry %s: %v", fullpath, err) @@ -81,7 +81,7 @@ func (store *AbstractSqlStore) DeleteEntry(fullpath filer2.FullPath) (*filer2.En dir, name := fullpath.DirAndName() res, err := store.DB.Exec("DELETE FROM filemeta WHERE dirhash=? AND name=? AND directory=?", - md5hash(dir), name, dir) + hashToLong(dir), name, dir) if err != nil { return nil, fmt.Errorf("mysql delete %s: %s", fullpath, err) } @@ -101,7 +101,7 @@ func (store *AbstractSqlStore) ListDirectoryEntries(fullpath filer2.FullPath, st sqlText = "SELECT NAME, meta FROM filemeta WHERE dirhash=? AND name>=? AND directory=? LIMIT ?" } - rows, err := store.DB.Query(sqlText, md5hash(string(fullpath)), startFileName, string(fullpath), limit) + rows, err := store.DB.Query(sqlText, hashToLong(string(fullpath)), startFileName, string(fullpath), limit) if err != nil { return nil, fmt.Errorf("mysql list %s : %v", fullpath, err) } diff --git a/weed/filer2/abstract_sql/hashing.go b/weed/filer2/abstract_sql/hashing.go index 5e1390233..5c982c537 100644 --- a/weed/filer2/abstract_sql/hashing.go +++ b/weed/filer2/abstract_sql/hashing.go @@ -5,9 +5,28 @@ import ( "io" ) -// returns a 128 bit hash -func md5hash(dir string) []byte { +// returns a 64 bit big int +func hashToLong(dir string) (v int64) { h := md5.New() io.WriteString(h, dir) - return h.Sum(nil) + + b := h.Sum(nil) + + v += int64(b[0]) + v <<= 8 + v += int64(b[1]) + v <<= 8 + v += int64(b[2]) + v <<= 8 + v += int64(b[3]) + v <<= 8 + v += int64(b[4]) + v <<= 8 + v += int64(b[5]) + v <<= 8 + v += int64(b[6]) + v <<= 8 + v += int64(b[7]) + + return } diff --git a/weed/filer2/configuration.go b/weed/filer2/configuration.go index 910ddcbbc..14ca61745 100644 --- a/weed/filer2/configuration.go +++ b/weed/filer2/configuration.go @@ -27,9 +27,9 @@ dir = "." # directory to store level db files # need to manually create a table "filemeta". # # CREATE TABLE IF NOT EXISTS filemeta ( -# dirhash BINARY(16) COMMENT 'MD5 hash value of directory field', -# name VARCHAR(1000) NOT NULL DEFAULT "" COMMENT 'directory or file name', -# directory VARCHAR(4096) NOT NULL DEFAULT "" COMMENT 'full path to parent directory', +# dirhash BIGINT COMMENT 'first 64 bits of MD5 hash value of directory field', +# name VARCHAR(1000) COMMENT 'directory or file name', +# directory VARCHAR(4096) COMMENT 'full path to parent directory', # meta BLOB, # PRIMARY KEY (dirhash, name) # ) DEFAULT CHARSET=utf8; @@ -44,10 +44,17 @@ connection_max_idle = 2 connection_max_open = 100 [postgres] +# CREATE TABLE IF NOT EXISTS filemeta ( +# dirhash BIGINT, +# name VARCHAR(1000), +# directory VARCHAR(4096), +# meta bytea, +# PRIMARY KEY (dirhash, name) +# ); enabled = false -server = "192.168.1.1" -port = 8080 -username = "" +server = "localhost" +port = 5432 +username = "postgres" password = "" database = "" connection_max_idle = 100 From 869161a2613a64bc5cf15f7841607f2721b8ef40 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sat, 26 May 2018 22:02:49 -0700 Subject: [PATCH 46/83] support both mysql and postgres --- .../filer2/abstract_sql/abstract_sql_store.go | 54 ++++++++------- weed/filer2/configuration.go | 9 +-- weed/filer2/mysql/mysql_store.go | 8 +++ weed/filer2/postgres/README.txt | 17 +++++ weed/filer2/postgres/postgres_store.go | 68 +++++++++++++++++++ weed/server/filer_server.go | 1 + 6 files changed, 127 insertions(+), 30 deletions(-) create mode 100644 weed/filer2/postgres/README.txt create mode 100644 weed/filer2/postgres/postgres_store.go diff --git a/weed/filer2/abstract_sql/abstract_sql_store.go b/weed/filer2/abstract_sql/abstract_sql_store.go index 900056dae..bfc76fbc0 100644 --- a/weed/filer2/abstract_sql/abstract_sql_store.go +++ b/weed/filer2/abstract_sql/abstract_sql_store.go @@ -9,7 +9,13 @@ import ( ) type AbstractSqlStore struct { - DB *sql.DB + DB *sql.DB + SqlInsert string + SqlUpdate string + SqlFind string + SqlDelete string + SqlListExclusive string + SqlListInclusive string } func (store *AbstractSqlStore) InsertEntry(entry *filer2.Entry) (err error) { @@ -17,18 +23,17 @@ func (store *AbstractSqlStore) InsertEntry(entry *filer2.Entry) (err error) { dir, name := entry.FullPath.DirAndName() meta, err := entry.EncodeAttributesAndChunks() if err != nil { - return fmt.Errorf("mysql encode %s: %s", entry.FullPath, err) + return fmt.Errorf("encode %s: %s", entry.FullPath, err) } - res, err := store.DB.Exec("INSERT INTO filemeta (dirhash,name,directory,meta) VALUES(?,?,?,?)", - hashToLong(dir), name, dir, meta) + res, err := store.DB.Exec(store.SqlInsert, hashToLong(dir), name, dir, meta) if err != nil { - return fmt.Errorf("mysql insert %s: %s", entry.FullPath, err) + return fmt.Errorf("insert %s: %s", entry.FullPath, err) } _, err = res.RowsAffected() if err != nil { - return fmt.Errorf("mysql insert %s but no rows affected: %s", entry.FullPath, err) + return fmt.Errorf("insert %s but no rows affected: %s", entry.FullPath, err) } return nil } @@ -38,18 +43,17 @@ func (store *AbstractSqlStore) UpdateEntry(entry *filer2.Entry) (err error) { dir, name := entry.FullPath.DirAndName() meta, err := entry.EncodeAttributesAndChunks() if err != nil { - return fmt.Errorf("mysql encode %s: %s", entry.FullPath, err) + return fmt.Errorf("encode %s: %s", entry.FullPath, err) } - res, err := store.DB.Exec("UPDATE filemeta SET meta=? WHERE dirhash=? AND name=? AND directory=?", - meta, hashToLong(dir), name, dir) + res, err := store.DB.Exec(store.SqlUpdate, meta, hashToLong(dir), name, dir) if err != nil { - return fmt.Errorf("mysql update %s: %s", entry.FullPath, err) + return fmt.Errorf("update %s: %s", entry.FullPath, err) } _, err = res.RowsAffected() if err != nil { - return fmt.Errorf("mysql update %s but no rows affected: %s", entry.FullPath, err) + return fmt.Errorf("update %s but no rows affected: %s", entry.FullPath, err) } return nil } @@ -57,18 +61,17 @@ func (store *AbstractSqlStore) UpdateEntry(entry *filer2.Entry) (err error) { func (store *AbstractSqlStore) FindEntry(fullpath filer2.FullPath) (*filer2.Entry, error) { dir, name := fullpath.DirAndName() - row := store.DB.QueryRow("SELECT meta FROM filemeta WHERE dirhash=? AND name=? AND directory=?", - hashToLong(dir), name, dir) + row := store.DB.QueryRow(store.SqlFind, hashToLong(dir), name, dir) var data []byte if err := row.Scan(&data); err != nil { - return nil, fmt.Errorf("mysql read entry %s: %v", fullpath, err) + return nil, fmt.Errorf("read entry %s: %v", fullpath, err) } entry := &filer2.Entry{ FullPath: fullpath, } if err := entry.DecodeAttributesAndChunks(data); err != nil { - return entry, fmt.Errorf("mysql decode %s : %v", entry.FullPath, err) + return entry, fmt.Errorf("decode %s : %v", entry.FullPath, err) } return entry, nil @@ -80,15 +83,14 @@ func (store *AbstractSqlStore) DeleteEntry(fullpath filer2.FullPath) (*filer2.En dir, name := fullpath.DirAndName() - res, err := store.DB.Exec("DELETE FROM filemeta WHERE dirhash=? AND name=? AND directory=?", - hashToLong(dir), name, dir) + res, err := store.DB.Exec(store.SqlDelete, hashToLong(dir), name, dir) if err != nil { - return nil, fmt.Errorf("mysql delete %s: %s", fullpath, err) + return nil, fmt.Errorf("delete %s: %s", fullpath, err) } _, err = res.RowsAffected() if err != nil { - return nil, fmt.Errorf("mysql delete %s but no rows affected: %s", fullpath, err) + return nil, fmt.Errorf("delete %s but no rows affected: %s", fullpath, err) } return entry, nil @@ -96,14 +98,14 @@ func (store *AbstractSqlStore) DeleteEntry(fullpath filer2.FullPath) (*filer2.En func (store *AbstractSqlStore) ListDirectoryEntries(fullpath filer2.FullPath, startFileName string, inclusive bool, limit int) (entries []*filer2.Entry, err error) { - sqlText := "SELECT NAME, meta FROM filemeta WHERE dirhash=? AND name>? AND directory=? LIMIT ?" + sqlText := store.SqlListExclusive if inclusive { - sqlText = "SELECT NAME, meta FROM filemeta WHERE dirhash=? AND name>=? AND directory=? LIMIT ?" + sqlText = store.SqlListInclusive } rows, err := store.DB.Query(sqlText, hashToLong(string(fullpath)), startFileName, string(fullpath), limit) if err != nil { - return nil, fmt.Errorf("mysql list %s : %v", fullpath, err) + return nil, fmt.Errorf("list %s : %v", fullpath, err) } defer rows.Close() @@ -111,16 +113,16 @@ func (store *AbstractSqlStore) ListDirectoryEntries(fullpath filer2.FullPath, st var name string var data []byte if err = rows.Scan(&name, &data); err != nil { - glog.V(0).Infof("mysql scan %s : %v", fullpath, err) - return nil, fmt.Errorf("mysql scan %s: %v", fullpath, err) + glog.V(0).Infof("scan %s : %v", fullpath, err) + return nil, fmt.Errorf("scan %s: %v", fullpath, err) } entry := &filer2.Entry{ FullPath: filer2.NewFullPath(string(fullpath), name), } if err = entry.DecodeAttributesAndChunks(data); err != nil { - glog.V(0).Infof("mysql scan decode %s : %v", entry.FullPath, err) - return nil, fmt.Errorf("mysql scan decode %s : %v", entry.FullPath, err) + glog.V(0).Infof("scan decode %s : %v", entry.FullPath, err) + return nil, fmt.Errorf("scan decode %s : %v", entry.FullPath, err) } entries = append(entries, entry) diff --git a/weed/filer2/configuration.go b/weed/filer2/configuration.go index 14ca61745..91a0d6f46 100644 --- a/weed/filer2/configuration.go +++ b/weed/filer2/configuration.go @@ -35,11 +35,11 @@ dir = "." # directory to store level db files # ) DEFAULT CHARSET=utf8; # enabled = true -server = "localhost" +hostname = "localhost" port = 3306 username = "root" password = "" -database = "" # create or use an existing database, create the seaweedfs table. +database = "" # create or use an existing database connection_max_idle = 2 connection_max_open = 100 @@ -52,11 +52,12 @@ connection_max_open = 100 # PRIMARY KEY (dirhash, name) # ); enabled = false -server = "localhost" +hostname = "localhost" port = 5432 username = "postgres" password = "" -database = "" +database = "" # create or use an existing database +sslmode = "disable" connection_max_idle = 100 connection_max_open = 100 diff --git a/weed/filer2/mysql/mysql_store.go b/weed/filer2/mysql/mysql_store.go index 16fd4311f..38afb99f1 100644 --- a/weed/filer2/mysql/mysql_store.go +++ b/weed/filer2/mysql/mysql_store.go @@ -39,6 +39,14 @@ func (store *MysqlStore) Initialize(viper *viper.Viper) (err error) { } func (store *MysqlStore) initialize(user, password, hostname string, port int, database string, maxIdle, maxOpen int) (err error) { + + store.SqlInsert = "INSERT INTO filemeta (dirhash,name,directory,meta) VALUES(?,?,?,?)" + store.SqlUpdate = "UPDATE filemeta SET meta=? WHERE dirhash=? AND name=? AND directory=?" + store.SqlFind = "SELECT meta FROM filemeta WHERE dirhash=? AND name=? AND directory=?" + store.SqlDelete = "DELETE FROM filemeta WHERE dirhash=? AND name=? AND directory=?" + store.SqlListExclusive = "SELECT NAME, meta FROM filemeta WHERE dirhash=? AND name>? AND directory=? LIMIT ?" + store.SqlListInclusive = "SELECT NAME, meta FROM filemeta WHERE dirhash=? AND name>=? AND directory=? LIMIT ?" + sqlUrl := fmt.Sprintf(CONNECTION_URL_PATTERN, user, password, hostname, port, database) var dbErr error store.DB, dbErr = sql.Open("mysql", sqlUrl) diff --git a/weed/filer2/postgres/README.txt b/weed/filer2/postgres/README.txt new file mode 100644 index 000000000..ef2ef683b --- /dev/null +++ b/weed/filer2/postgres/README.txt @@ -0,0 +1,17 @@ + +1. create "seaweedfs" database + +export PGHOME=/Library/PostgreSQL/10 +$PGHOME/bin/createdb --username=postgres --password seaweedfs + +2. create "filemeta" table +$PGHOME/bin/psql --username=postgres --password seaweedfs + +CREATE TABLE IF NOT EXISTS filemeta ( + dirhash BIGINT, + name VARCHAR(1000), + directory VARCHAR(4096), + meta bytea, + PRIMARY KEY (dirhash, name) +); + diff --git a/weed/filer2/postgres/postgres_store.go b/weed/filer2/postgres/postgres_store.go new file mode 100644 index 000000000..387b8f58d --- /dev/null +++ b/weed/filer2/postgres/postgres_store.go @@ -0,0 +1,68 @@ +package postgres + +import ( + "fmt" + "database/sql" + + "github.com/chrislusf/seaweedfs/weed/filer2" + "github.com/spf13/viper" + _ "github.com/lib/pq" + "github.com/chrislusf/seaweedfs/weed/filer2/abstract_sql" +) + +const ( + CONNECTION_URL_PATTERN = "host=%s port=%d user=%s password=%s dbname=%s sslmode=%s connect_timeout=30" +) + +func init() { + filer2.Stores = append(filer2.Stores, &PostgresStore{}) +} + +type PostgresStore struct { + abstract_sql.AbstractSqlStore +} + +func (store *PostgresStore) GetName() string { + return "postgres" +} + +func (store *PostgresStore) Initialize(viper *viper.Viper) (err error) { + return store.initialize( + viper.GetString("username"), + viper.GetString("password"), + viper.GetString("hostname"), + viper.GetInt("port"), + viper.GetString("database"), + viper.GetString("sslmode"), + viper.GetInt("connection_max_idle"), + viper.GetInt("connection_max_open"), + ) +} + +func (store *PostgresStore) initialize(user, password, hostname string, port int, database, sslmode string, maxIdle, maxOpen int) (err error) { + + store.SqlInsert = "INSERT INTO filemeta (dirhash,name,directory,meta) VALUES($1,$2,$3,$4)" + store.SqlUpdate = "UPDATE filemeta SET meta=$1 WHERE dirhash=$2 AND name=$3 AND directory=$4" + store.SqlFind = "SELECT meta FROM filemeta WHERE dirhash=$1 AND name=$2 AND directory=$3" + store.SqlDelete = "DELETE FROM filemeta WHERE dirhash=$1 AND name=$2 AND directory=$3" + store.SqlListExclusive = "SELECT NAME, meta FROM filemeta WHERE dirhash=$1 AND name>$2 AND directory=$3 LIMIT $4" + store.SqlListInclusive = "SELECT NAME, meta FROM filemeta WHERE dirhash=$1 AND name>=$2 AND directory=$3 LIMIT $4" + + sqlUrl := fmt.Sprintf(CONNECTION_URL_PATTERN, hostname, port, user, password, database, sslmode) + var dbErr error + store.DB, dbErr = sql.Open("postgres", sqlUrl) + if dbErr != nil { + store.DB.Close() + store.DB = nil + return fmt.Errorf("can not connect to %s error:%v", sqlUrl, err) + } + + store.DB.SetMaxIdleConns(maxIdle) + store.DB.SetMaxOpenConns(maxOpen) + + if err = store.DB.Ping(); err != nil { + return fmt.Errorf("connect to %s error:%v", sqlUrl, err) + } + + return nil +} diff --git a/weed/server/filer_server.go b/weed/server/filer_server.go index 2e88252e9..4517733ae 100644 --- a/weed/server/filer_server.go +++ b/weed/server/filer_server.go @@ -15,6 +15,7 @@ import ( _ "github.com/chrislusf/seaweedfs/weed/filer2/leveldb" _ "github.com/chrislusf/seaweedfs/weed/filer2/memdb" _ "github.com/chrislusf/seaweedfs/weed/filer2/mysql" + _ "github.com/chrislusf/seaweedfs/weed/filer2/postgres" ) type FilerServer struct { From 1c0a2a7db82c2f327cf06d19feb2e4bcf05f6262 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sat, 26 May 2018 23:59:56 -0700 Subject: [PATCH 47/83] reformat --- weed/server/filer_grpc_server.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/weed/server/filer_grpc_server.go b/weed/server/filer_grpc_server.go index 6aa8d50ff..ffa24a0d8 100644 --- a/weed/server/filer_grpc_server.go +++ b/weed/server/filer_grpc_server.go @@ -1,15 +1,16 @@ package weed_server import ( + "fmt" "context" + "time" + "os" + "path/filepath" + "github.com/chrislusf/seaweedfs/weed/operation" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" - "fmt" "github.com/chrislusf/seaweedfs/weed/filer2" - "path/filepath" "github.com/chrislusf/seaweedfs/weed/glog" - "time" - "os" ) func (fs *FilerServer) LookupDirectoryEntry(ctx context.Context, req *filer_pb.LookupDirectoryEntryRequest) (*filer_pb.LookupDirectoryEntryResponse, error) { From f124ebab1dd305687d897b50760647088a5d7985 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sun, 27 May 2018 00:00:11 -0700 Subject: [PATCH 48/83] fix ordering --- weed/filer2/mysql/mysql_store.go | 4 ++-- weed/filer2/postgres/postgres_store.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/weed/filer2/mysql/mysql_store.go b/weed/filer2/mysql/mysql_store.go index 38afb99f1..e56b961bb 100644 --- a/weed/filer2/mysql/mysql_store.go +++ b/weed/filer2/mysql/mysql_store.go @@ -44,8 +44,8 @@ func (store *MysqlStore) initialize(user, password, hostname string, port int, d store.SqlUpdate = "UPDATE filemeta SET meta=? WHERE dirhash=? AND name=? AND directory=?" store.SqlFind = "SELECT meta FROM filemeta WHERE dirhash=? AND name=? AND directory=?" store.SqlDelete = "DELETE FROM filemeta WHERE dirhash=? AND name=? AND directory=?" - store.SqlListExclusive = "SELECT NAME, meta FROM filemeta WHERE dirhash=? AND name>? AND directory=? LIMIT ?" - store.SqlListInclusive = "SELECT NAME, meta FROM filemeta WHERE dirhash=? AND name>=? AND directory=? LIMIT ?" + store.SqlListExclusive = "SELECT NAME, meta FROM filemeta WHERE dirhash=? AND name>? AND directory=? ORDER BY NAME ASC LIMIT ?" + store.SqlListInclusive = "SELECT NAME, meta FROM filemeta WHERE dirhash=? AND name>=? AND directory=? ORDER BY NAME ASC LIMIT ?" sqlUrl := fmt.Sprintf(CONNECTION_URL_PATTERN, user, password, hostname, port, database) var dbErr error diff --git a/weed/filer2/postgres/postgres_store.go b/weed/filer2/postgres/postgres_store.go index 387b8f58d..19d97b443 100644 --- a/weed/filer2/postgres/postgres_store.go +++ b/weed/filer2/postgres/postgres_store.go @@ -45,8 +45,8 @@ func (store *PostgresStore) initialize(user, password, hostname string, port int store.SqlUpdate = "UPDATE filemeta SET meta=$1 WHERE dirhash=$2 AND name=$3 AND directory=$4" store.SqlFind = "SELECT meta FROM filemeta WHERE dirhash=$1 AND name=$2 AND directory=$3" store.SqlDelete = "DELETE FROM filemeta WHERE dirhash=$1 AND name=$2 AND directory=$3" - store.SqlListExclusive = "SELECT NAME, meta FROM filemeta WHERE dirhash=$1 AND name>$2 AND directory=$3 LIMIT $4" - store.SqlListInclusive = "SELECT NAME, meta FROM filemeta WHERE dirhash=$1 AND name>=$2 AND directory=$3 LIMIT $4" + store.SqlListExclusive = "SELECT NAME, meta FROM filemeta WHERE dirhash=$1 AND name>$2 AND directory=$3 ORDER BY NAME ASC LIMIT $4" + store.SqlListInclusive = "SELECT NAME, meta FROM filemeta WHERE dirhash=$1 AND name>=$2 AND directory=$3 ORDER BY NAME ASC LIMIT $4" sqlUrl := fmt.Sprintf(CONNECTION_URL_PATTERN, hostname, port, user, password, database, sslmode) var dbErr error From 603de2d5db1f5c17e73db2b46c34b4e1efcf9815 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sun, 27 May 2018 00:00:56 -0700 Subject: [PATCH 49/83] skip deletion if entry not found --- weed/filer2/abstract_sql/abstract_sql_store.go | 5 ++++- weed/filer2/leveldb/leveldb_store.go | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/weed/filer2/abstract_sql/abstract_sql_store.go b/weed/filer2/abstract_sql/abstract_sql_store.go index bfc76fbc0..e924fa16a 100644 --- a/weed/filer2/abstract_sql/abstract_sql_store.go +++ b/weed/filer2/abstract_sql/abstract_sql_store.go @@ -79,7 +79,10 @@ func (store *AbstractSqlStore) FindEntry(fullpath filer2.FullPath) (*filer2.Entr func (store *AbstractSqlStore) DeleteEntry(fullpath filer2.FullPath) (*filer2.Entry, error) { - entry, _ := store.FindEntry(fullpath) + entry, err := store.FindEntry(fullpath) + if err != nil { + return nil, nil + } dir, name := fullpath.DirAndName() diff --git a/weed/filer2/leveldb/leveldb_store.go b/weed/filer2/leveldb/leveldb_store.go index 00fb44d36..8b2df93ac 100644 --- a/weed/filer2/leveldb/leveldb_store.go +++ b/weed/filer2/leveldb/leveldb_store.go @@ -96,7 +96,10 @@ func (store *LevelDBStore) FindEntry(fullpath filer2.FullPath) (entry *filer2.En func (store *LevelDBStore) DeleteEntry(fullpath filer2.FullPath) (entry *filer2.Entry, err error) { key := genKey(fullpath.DirAndName()) - entry, _ = store.FindEntry(fullpath) + entry, err = store.FindEntry(fullpath) + if err != nil { + return nil, nil + } err = store.db.Delete(key, nil) if err != nil { From ab86d263c91d496e2659f494ace9749392f157a4 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sun, 27 May 2018 00:01:15 -0700 Subject: [PATCH 50/83] add Cassandra --- weed/filer2/cassandra/README.txt | 12 ++ weed/filer2/cassandra/cassandra_store.go | 136 +++++++++++++++++++++++ weed/filer2/configuration.go | 13 +++ weed/server/filer_server.go | 1 + 4 files changed, 162 insertions(+) create mode 100644 weed/filer2/cassandra/README.txt create mode 100644 weed/filer2/cassandra/cassandra_store.go diff --git a/weed/filer2/cassandra/README.txt b/weed/filer2/cassandra/README.txt new file mode 100644 index 000000000..2d176229f --- /dev/null +++ b/weed/filer2/cassandra/README.txt @@ -0,0 +1,12 @@ +1. create a keyspace + +CREATE KEYSPACE seaweedfs WITH replication = {'class':'SimpleStrategy', 'replication_factor' : 1}; + +2. create filemeta table + + CREATE TABLE filemeta ( + directory varchar, + name varchar, + meta blob, + PRIMARY KEY (directory, name) + ) WITH CLUSTERING ORDER BY (name ASC); diff --git a/weed/filer2/cassandra/cassandra_store.go b/weed/filer2/cassandra/cassandra_store.go new file mode 100644 index 000000000..316b6cce0 --- /dev/null +++ b/weed/filer2/cassandra/cassandra_store.go @@ -0,0 +1,136 @@ +package cassandra + +import ( + "fmt" + "github.com/gocql/gocql" + "github.com/chrislusf/seaweedfs/weed/filer2" + "github.com/chrislusf/seaweedfs/weed/glog" + "github.com/spf13/viper" +) + +func init() { + filer2.Stores = append(filer2.Stores, &CassandraStore{}) +} + +type CassandraStore struct { + cluster *gocql.ClusterConfig + session *gocql.Session +} + +func (store *CassandraStore) GetName() string { + return "cassandra" +} + +func (store *CassandraStore) Initialize(viper *viper.Viper) (err error) { + return store.initialize( + viper.GetString("keyspace"), + viper.GetStringSlice("hosts"), + ) +} + +func (store *CassandraStore) initialize(keyspace string, hosts []string) (err error) { + store.cluster = gocql.NewCluster(hosts...) + store.cluster.Keyspace = keyspace + store.cluster.Consistency = gocql.LocalQuorum + store.session, err = store.cluster.CreateSession() + if err != nil { + glog.V(0).Infof("Failed to open cassandra store, hosts %v, keyspace %s", hosts, keyspace) + } + return +} + +func (store *CassandraStore) InsertEntry(entry *filer2.Entry) (err error) { + + dir, name := entry.FullPath.DirAndName() + meta, err := entry.EncodeAttributesAndChunks() + if err != nil { + return fmt.Errorf("encode %s: %s", entry.FullPath, err) + } + + if err := store.session.Query( + "INSERT INTO filemeta (directory,name,meta) VALUES(?,?,?)", + dir, name, meta).Exec(); err != nil { + return fmt.Errorf("insert %s: %s", entry.FullPath, err) + } + + return nil +} + +func (store *CassandraStore) UpdateEntry(entry *filer2.Entry) (err error) { + + return store.InsertEntry(entry) +} + +func (store *CassandraStore) FindEntry(fullpath filer2.FullPath) (entry *filer2.Entry, err error) { + + dir, name := fullpath.DirAndName() + var data []byte + if err := store.session.Query( + "SELECT meta FROM filemeta WHERE directory=? AND name=?", + dir, name).Consistency(gocql.One).Scan(&data); err != nil { + if err != gocql.ErrNotFound { + return nil, fmt.Errorf("read entry %s: %v", fullpath, err) + } + } + + if len(data) == 0 { + return nil, fmt.Errorf("not found: %s", fullpath) + } + + entry = &filer2.Entry{ + FullPath: fullpath, + } + err = entry.DecodeAttributesAndChunks(data) + if err != nil { + return entry, fmt.Errorf("decode %s : %v", entry.FullPath, err) + } + + return entry, nil +} + +func (store *CassandraStore) DeleteEntry(fullpath filer2.FullPath) (entry *filer2.Entry, err error) { + + entry, err = store.FindEntry(fullpath) + if err != nil { + return nil, nil + } + + dir, name := fullpath.DirAndName() + + if err := store.session.Query( + "DELETE FROM filemeta WHERE directory=? AND name=?", + dir, name).Exec(); err != nil { + return entry, fmt.Errorf("delete %s : %v", entry.FullPath, err) + } + + return entry, nil +} + +func (store *CassandraStore) ListDirectoryEntries(fullpath filer2.FullPath, startFileName string, inclusive bool, + limit int) (entries []*filer2.Entry, err error) { + + cqlStr := "SELECT NAME, meta FROM filemeta WHERE directory=? AND name>? ORDER BY NAME ASC LIMIT ?" + if inclusive { + cqlStr = "SELECT NAME, meta FROM filemeta WHERE directory=? AND name>=? ORDER BY NAME ASC LIMIT ?" + } + + var data []byte + var name string + iter := store.session.Query(cqlStr, string(fullpath), startFileName, limit).Iter() + for iter.Scan(&name, &data) { + entry := &filer2.Entry{ + FullPath: filer2.NewFullPath(string(fullpath), name), + } + if decodeErr := entry.DecodeAttributesAndChunks(data); decodeErr != nil { + err = decodeErr + glog.V(0).Infof("list %s : %v", entry.FullPath, err) + break + } + entries = append(entries, entry) + } + if err := iter.Close(); err != nil { + glog.V(0).Infof("list iterator close: %v", err) + } + + return entries, err +} diff --git a/weed/filer2/configuration.go b/weed/filer2/configuration.go index 91a0d6f46..0112a883b 100644 --- a/weed/filer2/configuration.go +++ b/weed/filer2/configuration.go @@ -61,6 +61,19 @@ sslmode = "disable" connection_max_idle = 100 connection_max_open = 100 +[cassandra] +# CREATE TABLE filemeta ( +# directory varchar, +# name varchar, +# meta blob, +# PRIMARY KEY (directory, name) +# ) WITH CLUSTERING ORDER BY (name ASC); +enabled = false +keyspace="seaweedfs" +hosts=[ + "localhost:9042", +] + ` ) diff --git a/weed/server/filer_server.go b/weed/server/filer_server.go index 4517733ae..6c8a2c079 100644 --- a/weed/server/filer_server.go +++ b/weed/server/filer_server.go @@ -12,6 +12,7 @@ import ( "github.com/chrislusf/seaweedfs/weed/storage" "github.com/chrislusf/seaweedfs/weed/util" "github.com/chrislusf/seaweedfs/weed/filer2" + _ "github.com/chrislusf/seaweedfs/weed/filer2/cassandra" _ "github.com/chrislusf/seaweedfs/weed/filer2/leveldb" _ "github.com/chrislusf/seaweedfs/weed/filer2/memdb" _ "github.com/chrislusf/seaweedfs/weed/filer2/mysql" From a6a6b87f1baacdf51176b716b72fe61818d548d3 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sun, 27 May 2018 11:14:12 -0700 Subject: [PATCH 51/83] adjust readme --- weed/filer2/cassandra/README.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/weed/filer2/cassandra/README.txt b/weed/filer2/cassandra/README.txt index 2d176229f..122c9c3f4 100644 --- a/weed/filer2/cassandra/README.txt +++ b/weed/filer2/cassandra/README.txt @@ -4,6 +4,8 @@ CREATE KEYSPACE seaweedfs WITH replication = {'class':'SimpleStrategy', 'replica 2. create filemeta table + USE seaweedfs; + CREATE TABLE filemeta ( directory varchar, name varchar, From 03d76479a57a4e28a56d89b14d78300756992007 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sun, 27 May 2018 11:14:29 -0700 Subject: [PATCH 52/83] add reds --- weed/filer2/configuration.go | 6 ++ weed/filer2/redis/redis_store.go | 172 +++++++++++++++++++++++++++++++ weed/server/filer_server.go | 1 + 3 files changed, 179 insertions(+) create mode 100644 weed/filer2/redis/redis_store.go diff --git a/weed/filer2/configuration.go b/weed/filer2/configuration.go index 0112a883b..16c0f3fd0 100644 --- a/weed/filer2/configuration.go +++ b/weed/filer2/configuration.go @@ -74,6 +74,12 @@ hosts=[ "localhost:9042", ] +[redis] +enabled = true +address = "localhost:6379" +password = "" +db = 0 + ` ) diff --git a/weed/filer2/redis/redis_store.go b/weed/filer2/redis/redis_store.go new file mode 100644 index 000000000..cb8381bf5 --- /dev/null +++ b/weed/filer2/redis/redis_store.go @@ -0,0 +1,172 @@ +package redis + +import ( + "fmt" + "github.com/chrislusf/seaweedfs/weed/filer2" + "github.com/chrislusf/seaweedfs/weed/glog" + "github.com/spf13/viper" + "github.com/go-redis/redis" + "sort" + "strings" +) + +const ( + DIR_LIST_MARKER = "\x00" +) + +func init() { + filer2.Stores = append(filer2.Stores, &RedisStore{}) +} + +type RedisStore struct { + Client *redis.Client +} + +func (store *RedisStore) GetName() string { + return "redis" +} + +func (store *RedisStore) Initialize(viper *viper.Viper) (err error) { + return store.initialize( + viper.GetString("address"), + viper.GetString("password"), + viper.GetInt("database"), + ) +} + +func (store *RedisStore) initialize(hostPort string, password string, database int) (err error) { + store.Client = redis.NewClient(&redis.Options{ + Addr: hostPort, + Password: password, + DB: database, + }) + return +} + +func (store *RedisStore) InsertEntry(entry *filer2.Entry) (err error) { + + value, err := entry.EncodeAttributesAndChunks() + if err != nil { + return fmt.Errorf("encoding %s %+v: %v", entry.FullPath, entry.Attr, err) + } + + _, err = store.Client.Set(string(entry.FullPath), value, 0).Result() + + if err != nil { + return fmt.Errorf("persisting %s : %v", entry.FullPath, err) + } + + dir, name := entry.FullPath.DirAndName() + if name != "" { + _, err = store.Client.SAdd(genDirectoryListKey(dir), name).Result() + if err != nil { + return fmt.Errorf("persisting %s in parent dir: %v", entry.FullPath, err) + } + } + + return nil +} + +func (store *RedisStore) UpdateEntry(entry *filer2.Entry) (err error) { + + return store.InsertEntry(entry) +} + +func (store *RedisStore) FindEntry(fullpath filer2.FullPath) (entry *filer2.Entry, err error) { + + data, err := store.Client.Get(string(fullpath)).Result() + if err == redis.Nil { + return nil, filer2.ErrNotFound + } + + if err != nil { + return nil, fmt.Errorf("get %s : %v", entry.FullPath, err) + } + + entry = &filer2.Entry{ + FullPath: fullpath, + } + err = entry.DecodeAttributesAndChunks([]byte(data)) + if err != nil { + return entry, fmt.Errorf("decode %s : %v", entry.FullPath, err) + } + + return entry, nil +} + +func (store *RedisStore) DeleteEntry(fullpath filer2.FullPath) (entry *filer2.Entry, err error) { + + entry, err = store.FindEntry(fullpath) + if err != nil { + return nil, nil + } + + _, err = store.Client.Del(string(fullpath)).Result() + + if err != nil { + return entry, fmt.Errorf("delete %s : %v", entry.FullPath, err) + } + + dir, name := fullpath.DirAndName() + if name != "" { + _, err = store.Client.SRem(genDirectoryListKey(dir), name).Result() + if err != nil { + return nil, fmt.Errorf("delete %s in parent dir: %v", entry.FullPath, err) + } + } + + return entry, nil +} + +func (store *RedisStore) ListDirectoryEntries(fullpath filer2.FullPath, startFileName string, inclusive bool, + limit int) (entries []*filer2.Entry, err error) { + + members, err := store.Client.SMembers(genDirectoryListKey(string(fullpath))).Result() + if err != nil { + return nil, fmt.Errorf("list %s : %v", fullpath, err) + } + + // skip + if startFileName != "" { + var t []string + for _, m := range members { + if strings.Compare(m, startFileName) >= 0 { + if m == startFileName { + if inclusive { + t = append(t, m) + } + } else { + t = append(t, m) + } + } + } + members = t + } + + // sort + sort.Slice(members, func(i, j int) bool { + return strings.Compare(members[i], members[j]) < 0 + }) + + // limit + if limit < len(members) { + members = members[:limit] + } + + // fetch entry meta + for _, fileName := range members { + path := filer2.NewFullPath(string(fullpath), fileName) + entry, err := store.FindEntry(path) + if err != nil { + glog.V(0).Infof("list %s : %v", path, err) + } else { + entries = append(entries, entry) + } + } + + return entries, err +} + +func genDirectoryListKey(dir string) (dirList string) { + return dir + DIR_LIST_MARKER +} diff --git a/weed/server/filer_server.go b/weed/server/filer_server.go index 6c8a2c079..012b0afbf 100644 --- a/weed/server/filer_server.go +++ b/weed/server/filer_server.go @@ -17,6 +17,7 @@ import ( _ "github.com/chrislusf/seaweedfs/weed/filer2/memdb" _ "github.com/chrislusf/seaweedfs/weed/filer2/mysql" _ "github.com/chrislusf/seaweedfs/weed/filer2/postgres" + _ "github.com/chrislusf/seaweedfs/weed/filer2/redis" ) type FilerServer struct { From 8647191bee2027232105b20b5c110ffda225f9ba Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sun, 27 May 2018 11:52:15 -0700 Subject: [PATCH 53/83] removing filer package! --- weed/filer/cassandra_store/cassandra_store.go | 96 ---- weed/filer/cassandra_store/schema.cql | 22 - weed/filer/embedded_filer/design.txt | 26 - weed/filer/embedded_filer/directory.go | 15 - weed/filer/embedded_filer/directory_in_map.go | 312 ------------ weed/filer/embedded_filer/directory_test.go | 86 ---- weed/filer/embedded_filer/filer_embedded.go | 156 ------ weed/filer/embedded_filer/files_in_leveldb.go | 87 ---- weed/filer/filer.go | 29 -- .../flat_namespace/flat_namespace_filer.go | 66 --- .../flat_namespace/flat_namespace_store.go | 9 - weed/filer/mysql_store/README.md | 67 --- weed/filer/mysql_store/mysql_store.go | 270 ----------- weed/filer/mysql_store/mysql_store_test.go | 30 -- weed/filer/postgres_store/postgres_native.go | 456 ------------------ weed/filer/postgres_store/postgres_store.go | 149 ------ weed/filer/redis_store/redis_store.go | 50 -- weed/filer/vasto_store/design.txt | 45 -- weed/server/filer_server_handlers_write.go | 2 +- 19 files changed, 1 insertion(+), 1972 deletions(-) delete mode 100644 weed/filer/cassandra_store/cassandra_store.go delete mode 100644 weed/filer/cassandra_store/schema.cql delete mode 100644 weed/filer/embedded_filer/design.txt delete mode 100644 weed/filer/embedded_filer/directory.go delete mode 100644 weed/filer/embedded_filer/directory_in_map.go delete mode 100644 weed/filer/embedded_filer/directory_test.go delete mode 100644 weed/filer/embedded_filer/filer_embedded.go delete mode 100644 weed/filer/embedded_filer/files_in_leveldb.go delete mode 100644 weed/filer/filer.go delete mode 100644 weed/filer/flat_namespace/flat_namespace_filer.go delete mode 100644 weed/filer/flat_namespace/flat_namespace_store.go delete mode 100644 weed/filer/mysql_store/README.md delete mode 100644 weed/filer/mysql_store/mysql_store.go delete mode 100644 weed/filer/mysql_store/mysql_store_test.go delete mode 100644 weed/filer/postgres_store/postgres_native.go delete mode 100644 weed/filer/postgres_store/postgres_store.go delete mode 100644 weed/filer/redis_store/redis_store.go delete mode 100644 weed/filer/vasto_store/design.txt diff --git a/weed/filer/cassandra_store/cassandra_store.go b/weed/filer/cassandra_store/cassandra_store.go deleted file mode 100644 index 837fa48d3..000000000 --- a/weed/filer/cassandra_store/cassandra_store.go +++ /dev/null @@ -1,96 +0,0 @@ -package cassandra_store - -import ( - "fmt" - "strings" - - "github.com/chrislusf/seaweedfs/weed/filer" - "github.com/chrislusf/seaweedfs/weed/glog" - - "github.com/gocql/gocql" -) - -/* - -Basically you need a table just like this: - -CREATE TABLE seaweed_files ( - path varchar, - fids list, - PRIMARY KEY (path) -); - -Need to match flat_namespace.FlatNamespaceStore interface - Put(fullFileName string, fid string) (err error) - Get(fullFileName string) (fid string, err error) - Delete(fullFileName string) (fid string, err error) - -*/ -type CassandraStore struct { - cluster *gocql.ClusterConfig - session *gocql.Session -} - -func NewCassandraStore(keyspace string, hosts string) (c *CassandraStore, err error) { - c = &CassandraStore{} - s := strings.Split(hosts, ",") - if len(s) == 1 { - glog.V(2).Info("Only one cassandra node to connect! A cluster is Recommended! Now using:", string(hosts)) - c.cluster = gocql.NewCluster(hosts) - } else if len(s) > 1 { - c.cluster = gocql.NewCluster(s...) - } - c.cluster.Keyspace = keyspace - c.cluster.Consistency = gocql.LocalQuorum - c.session, err = c.cluster.CreateSession() - if err != nil { - glog.V(0).Infof("Failed to open cassandra store, hosts %v, keyspace %s", hosts, keyspace) - } - return -} - -func (c *CassandraStore) Put(fullFileName string, fid string) (err error) { - var input []string - input = append(input, fid) - if err := c.session.Query( - `INSERT INTO seaweed_files (path, fids) VALUES (?, ?)`, - fullFileName, input).Exec(); err != nil { - glog.V(0).Infof("Failed to save file %s with id %s: %v", fullFileName, fid, err) - return err - } - return nil -} -func (c *CassandraStore) Get(fullFileName string) (fid string, err error) { - var output []string - if err := c.session.Query( - `select fids FROM seaweed_files WHERE path = ? LIMIT 1`, - fullFileName).Consistency(gocql.One).Scan(&output); err != nil { - if err != gocql.ErrNotFound { - glog.V(0).Infof("Failed to find file %s: %v", fullFileName, fid, err) - return "", filer.ErrNotFound - } - } - if len(output) == 0 { - return "", fmt.Errorf("No file id found for %s", fullFileName) - } - return output[0], nil -} - -// Currently the fid is not returned -func (c *CassandraStore) Delete(fullFileName string) (err error) { - if err := c.session.Query( - `DELETE FROM seaweed_files WHERE path = ?`, - fullFileName).Exec(); err != nil { - if err != gocql.ErrNotFound { - glog.V(0).Infof("Failed to delete file %s: %v", fullFileName, err) - } - return err - } - return nil -} - -func (c *CassandraStore) Close() { - if c.session != nil { - c.session.Close() - } -} diff --git a/weed/filer/cassandra_store/schema.cql b/weed/filer/cassandra_store/schema.cql deleted file mode 100644 index d6f2bb093..000000000 --- a/weed/filer/cassandra_store/schema.cql +++ /dev/null @@ -1,22 +0,0 @@ -/* - -Here is the CQL to create the table.CassandraStore - -Optionally you can adjust the keyspace name and replication settings. - -For production server, very likely you want to set replication_factor to 3 - -*/ - -create keyspace seaweed WITH replication = { - 'class':'SimpleStrategy', - 'replication_factor':1 -}; - -use seaweed; - -CREATE TABLE seaweed_files ( - path varchar, - fids list, - PRIMARY KEY (path) -); diff --git a/weed/filer/embedded_filer/design.txt b/weed/filer/embedded_filer/design.txt deleted file mode 100644 index 45fec8fbe..000000000 --- a/weed/filer/embedded_filer/design.txt +++ /dev/null @@ -1,26 +0,0 @@ -Design Assumptions: -1. the number of directories are magnitudely smaller than the number of files -2. unlimited number of files under any directories -Phylosophy: - metadata for directories and files should be separated -Design: - Store directories in normal map - all of directories hopefully all be in memory - efficient to move/rename/list_directories - Log directory changes to append only log file - Store files in sorted string table in format - efficient to list_files, just simple iterator - efficient to locate files, binary search - -Testing: -1. starting server, "weed server -filer=true" -2. posting files to different folders -curl -F "filename=@design.txt" "http://localhost:8888/sources/" -curl -F "filename=@design.txt" "http://localhost:8888/design/" -curl -F "filename=@directory.go" "http://localhost:8888/sources/weed/go/" -curl -F "filename=@directory.go" "http://localhost:8888/sources/testing/go/" -curl -F "filename=@filer.go" "http://localhost:8888/sources/weed/go/" -curl -F "filename=@filer_in_leveldb.go" "http://localhost:8888/sources/weed/go/" -curl "http://localhost:8888/?pretty=y" -curl "http://localhost:8888/sources/weed/go/?pretty=y" -curl "http://localhost:8888/sources/weed/go/?pretty=y" diff --git a/weed/filer/embedded_filer/directory.go b/weed/filer/embedded_filer/directory.go deleted file mode 100644 index 1ff725f28..000000000 --- a/weed/filer/embedded_filer/directory.go +++ /dev/null @@ -1,15 +0,0 @@ -package embedded_filer - -import ( - "github.com/chrislusf/seaweedfs/weed/filer" -) - -type DirectoryManager interface { - FindDirectory(dirPath string) (DirectoryId, error) - ListDirectories(dirPath string) (dirs []filer.DirectoryName, err error) - MakeDirectory(currentDirPath string, dirName string) (DirectoryId, error) - MoveUnderDirectory(oldDirPath string, newParentDirPath string) error - DeleteDirectory(dirPath string) error - //functions used by FUSE - FindDirectoryById(DirectoryId, error) -} diff --git a/weed/filer/embedded_filer/directory_in_map.go b/weed/filer/embedded_filer/directory_in_map.go deleted file mode 100644 index 1266d3779..000000000 --- a/weed/filer/embedded_filer/directory_in_map.go +++ /dev/null @@ -1,312 +0,0 @@ -package embedded_filer - -import ( - "bufio" - "fmt" - "io" - "os" - "path/filepath" - "strconv" - "strings" - "sync" - - "github.com/chrislusf/seaweedfs/weed/filer" - "github.com/chrislusf/seaweedfs/weed/util" -) - -var writeLock sync.Mutex //serialize changes to dir.log - -type DirectoryId int32 - -type DirectoryEntryInMap struct { - sync.Mutex - Name string - Parent *DirectoryEntryInMap - subDirectories map[string]*DirectoryEntryInMap - Id DirectoryId -} - -func (de *DirectoryEntryInMap) getChild(dirName string) (*DirectoryEntryInMap, bool) { - de.Lock() - defer de.Unlock() - child, ok := de.subDirectories[dirName] - return child, ok -} -func (de *DirectoryEntryInMap) addChild(dirName string, child *DirectoryEntryInMap) { - de.Lock() - defer de.Unlock() - de.subDirectories[dirName] = child -} -func (de *DirectoryEntryInMap) removeChild(dirName string) { - de.Lock() - defer de.Unlock() - delete(de.subDirectories, dirName) -} -func (de *DirectoryEntryInMap) hasChildren() bool { - de.Lock() - defer de.Unlock() - return len(de.subDirectories) > 0 -} -func (de *DirectoryEntryInMap) children() (dirNames []filer.DirectoryName) { - de.Lock() - defer de.Unlock() - for k, _ := range de.subDirectories { - dirNames = append(dirNames, filer.DirectoryName(k)) - } - return dirNames -} - -type DirectoryManagerInMap struct { - Root *DirectoryEntryInMap - max DirectoryId - logFile *os.File - isLoading bool -} - -func (dm *DirectoryManagerInMap) newDirectoryEntryInMap(parent *DirectoryEntryInMap, name string) (d *DirectoryEntryInMap, err error) { - d = &DirectoryEntryInMap{Name: name, Parent: parent, subDirectories: make(map[string]*DirectoryEntryInMap)} - var parts []string - for p := d; p != nil && p.Name != ""; p = p.Parent { - parts = append(parts, p.Name) - } - n := len(parts) - if n <= 0 { - return nil, fmt.Errorf("Failed to create folder %s/%s", parent.Name, name) - } - for i := 0; i < n/2; i++ { - parts[i], parts[n-1-i] = parts[n-1-i], parts[i] - } - dm.max++ - d.Id = dm.max - dm.log("add", "/"+strings.Join(parts, "/"), strconv.Itoa(int(d.Id))) - return d, nil -} - -func (dm *DirectoryManagerInMap) log(words ...string) { - if !dm.isLoading { - dm.logFile.WriteString(strings.Join(words, "\t") + "\n") - } -} - -func NewDirectoryManagerInMap(dirLogFile string) (dm *DirectoryManagerInMap, err error) { - dm = &DirectoryManagerInMap{} - //dm.Root do not use newDirectoryEntryInMap, since dm.max will be changed - dm.Root = &DirectoryEntryInMap{subDirectories: make(map[string]*DirectoryEntryInMap)} - if dm.logFile, err = os.OpenFile(dirLogFile, os.O_RDWR|os.O_CREATE, 0644); err != nil { - return nil, fmt.Errorf("cannot write directory log file %s: %v", dirLogFile, err) - } - return dm, dm.load() -} - -func (dm *DirectoryManagerInMap) processEachLine(line string) error { - if strings.HasPrefix(line, "#") { - return nil - } - if line == "" { - return nil - } - parts := strings.Split(line, "\t") - if len(parts) == 0 { - return nil - } - switch parts[0] { - case "add": - v, pe := strconv.Atoi(parts[2]) - if pe != nil { - return pe - } - if e := dm.loadDirectory(parts[1], DirectoryId(v)); e != nil { - return e - } - case "mov": - newName := "" - if len(parts) >= 4 { - newName = parts[3] - } - if e := dm.MoveUnderDirectory(parts[1], parts[2], newName); e != nil { - return e - } - case "del": - if e := dm.DeleteDirectory(parts[1]); e != nil { - return e - } - default: - fmt.Printf("line %s has %s!\n", line, parts[0]) - return nil - } - return nil -} -func (dm *DirectoryManagerInMap) load() error { - dm.max = 0 - lines := bufio.NewReader(dm.logFile) - dm.isLoading = true - defer func() { dm.isLoading = false }() - for { - line, err := util.Readln(lines) - if err != nil && err != io.EOF { - return err - } - if pe := dm.processEachLine(string(line)); pe != nil { - return pe - } - if err == io.EOF { - return nil - } - } -} - -func (dm *DirectoryManagerInMap) findDirectory(dirPath string) (*DirectoryEntryInMap, error) { - if dirPath == "" { - return dm.Root, nil - } - dirPath = CleanFilePath(dirPath) - if dirPath == "/" { - return dm.Root, nil - } - parts := strings.Split(dirPath, "/") - dir := dm.Root - for i := 1; i < len(parts); i++ { - if sub, ok := dir.getChild(parts[i]); ok { - dir = sub - } else { - return dm.Root, filer.ErrNotFound - } - } - return dir, nil -} -func (dm *DirectoryManagerInMap) findDirectoryId(dirPath string) (DirectoryId, error) { - d, e := dm.findDirectory(dirPath) - if e == nil { - return d.Id, nil - } - return dm.Root.Id, e -} - -func (dm *DirectoryManagerInMap) loadDirectory(dirPath string, dirId DirectoryId) error { - dirPath = CleanFilePath(dirPath) - if dirPath == "/" { - return nil - } - parts := strings.Split(dirPath, "/") - dir := dm.Root - for i := 1; i < len(parts); i++ { - sub, ok := dir.getChild(parts[i]) - if !ok { - writeLock.Lock() - if sub2, createdByOtherThread := dir.getChild(parts[i]); createdByOtherThread { - sub = sub2 - } else { - if i != len(parts)-1 { - writeLock.Unlock() - return fmt.Errorf("%s should be created after parent %s", dirPath, parts[i]) - } - var err error - sub, err = dm.newDirectoryEntryInMap(dir, parts[i]) - if err != nil { - writeLock.Unlock() - return err - } - if sub.Id != dirId { - writeLock.Unlock() - // the dir.log should be the same order as in-memory directory id - return fmt.Errorf("%s should be have id %v instead of %v", dirPath, sub.Id, dirId) - } - dir.addChild(parts[i], sub) - } - writeLock.Unlock() - } - dir = sub - } - return nil -} - -func (dm *DirectoryManagerInMap) makeDirectory(dirPath string) (dir *DirectoryEntryInMap, created bool) { - dirPath = CleanFilePath(dirPath) - if dirPath == "/" { - return dm.Root, false - } - parts := strings.Split(dirPath, "/") - dir = dm.Root - for i := 1; i < len(parts); i++ { - sub, ok := dir.getChild(parts[i]) - if !ok { - writeLock.Lock() - if sub2, createdByOtherThread := dir.getChild(parts[i]); createdByOtherThread { - sub = sub2 - } else { - var err error - sub, err = dm.newDirectoryEntryInMap(dir, parts[i]) - if err != nil { - writeLock.Unlock() - return nil, false - } - dir.addChild(parts[i], sub) - created = true - } - writeLock.Unlock() - } - dir = sub - } - return dir, created -} - -func (dm *DirectoryManagerInMap) MakeDirectory(dirPath string) (DirectoryId, error) { - dir, _ := dm.makeDirectory(dirPath) - return dir.Id, nil -} - -func (dm *DirectoryManagerInMap) MoveUnderDirectory(oldDirPath string, newParentDirPath string, newName string) error { - writeLock.Lock() - defer writeLock.Unlock() - oldDir, oe := dm.findDirectory(oldDirPath) - if oe != nil { - return oe - } - parentDir, pe := dm.findDirectory(newParentDirPath) - if pe != nil { - return pe - } - dm.log("mov", oldDirPath, newParentDirPath, newName) - oldDir.Parent.removeChild(oldDir.Name) - if newName == "" { - newName = oldDir.Name - } - parentDir.addChild(newName, oldDir) - oldDir.Name = newName - oldDir.Parent = parentDir - return nil -} - -func (dm *DirectoryManagerInMap) ListDirectories(dirPath string) (dirNames []filer.DirectoryName, err error) { - d, e := dm.findDirectory(dirPath) - if e != nil { - return dirNames, e - } - return d.children(), nil -} -func (dm *DirectoryManagerInMap) DeleteDirectory(dirPath string) error { - writeLock.Lock() - defer writeLock.Unlock() - if dirPath == "/" { - return fmt.Errorf("Can not delete %s", dirPath) - } - d, e := dm.findDirectory(dirPath) - if e != nil { - return e - } - if d.hasChildren() { - return fmt.Errorf("dir %s still has sub directories", dirPath) - } - d.Parent.removeChild(d.Name) - d.Parent = nil - dm.log("del", dirPath) - return nil -} - -func CleanFilePath(fp string) string { - ret := filepath.Clean(fp) - if os.PathSeparator == '\\' { - return strings.Replace(ret, "\\", "/", -1) - } - return ret -} diff --git a/weed/filer/embedded_filer/directory_test.go b/weed/filer/embedded_filer/directory_test.go deleted file mode 100644 index d9c941510..000000000 --- a/weed/filer/embedded_filer/directory_test.go +++ /dev/null @@ -1,86 +0,0 @@ -package embedded_filer - -import ( - "os" - "strings" - "testing" -) - -func TestDirectory(t *testing.T) { - dm, _ := NewDirectoryManagerInMap("/tmp/dir.log") - defer func() { - if true { - os.Remove("/tmp/dir.log") - } - }() - dm.MakeDirectory("/a/b/c") - dm.MakeDirectory("/a/b/d") - dm.MakeDirectory("/a/b/e") - dm.MakeDirectory("/a/b/e/f") - dm.MakeDirectory("/a/b/e/f/g") - dm.MoveUnderDirectory("/a/b/e/f/g", "/a/b", "t") - if _, err := dm.findDirectoryId("/a/b/e/f/g"); err == nil { - t.Fatal("/a/b/e/f/g should not exist any more after moving") - } - if _, err := dm.findDirectoryId("/a/b/t"); err != nil { - t.Fatal("/a/b/t should exist after moving") - } - if _, err := dm.findDirectoryId("/a/b/g"); err == nil { - t.Fatal("/a/b/g should not exist after moving") - } - dm.MoveUnderDirectory("/a/b/e/f", "/a/b", "") - if _, err := dm.findDirectoryId("/a/b/f"); err != nil { - t.Fatal("/a/b/g should not exist after moving") - } - dm.MakeDirectory("/a/b/g/h/i") - dm.DeleteDirectory("/a/b/e/f") - dm.DeleteDirectory("/a/b/e") - dirNames, _ := dm.ListDirectories("/a/b/e") - for _, v := range dirNames { - println("sub1 dir:", v) - } - dm.logFile.Close() - - var path []string - printTree(dm.Root, path) - - dm2, e := NewDirectoryManagerInMap("/tmp/dir.log") - if e != nil { - println("load error", e.Error()) - } - if !compare(dm.Root, dm2.Root) { - t.Fatal("restored dir not the same!") - } - printTree(dm2.Root, path) -} - -func printTree(node *DirectoryEntryInMap, path []string) { - println(strings.Join(path, "/") + "/" + node.Name) - path = append(path, node.Name) - for _, v := range node.subDirectories { - printTree(v, path) - } -} - -func compare(root1 *DirectoryEntryInMap, root2 *DirectoryEntryInMap) bool { - if len(root1.subDirectories) != len(root2.subDirectories) { - return false - } - if root1.Name != root2.Name { - return false - } - if root1.Id != root2.Id { - return false - } - if !(root1.Parent == nil && root2.Parent == nil) { - if root1.Parent.Id != root2.Parent.Id { - return false - } - } - for k, v := range root1.subDirectories { - if !compare(v, root2.subDirectories[k]) { - return false - } - } - return true -} diff --git a/weed/filer/embedded_filer/filer_embedded.go b/weed/filer/embedded_filer/filer_embedded.go deleted file mode 100644 index d6ab3c70a..000000000 --- a/weed/filer/embedded_filer/filer_embedded.go +++ /dev/null @@ -1,156 +0,0 @@ -package embedded_filer - -import ( - "errors" - "fmt" - "path/filepath" - "strings" - "sync" - - "github.com/chrislusf/seaweedfs/weed/filer" - "github.com/chrislusf/seaweedfs/weed/operation" -) - -type FilerEmbedded struct { - master string - directories *DirectoryManagerInMap - files *FileListInLevelDb - mvMutex sync.Mutex -} - -func NewFilerEmbedded(master string, dir string) (filer *FilerEmbedded, err error) { - dm, de := NewDirectoryManagerInMap(filepath.Join(dir, "dir.log")) - if de != nil { - return nil, de - } - fl, fe := NewFileListInLevelDb(dir) - if fe != nil { - return nil, fe - } - filer = &FilerEmbedded{ - master: master, - directories: dm, - files: fl, - } - return -} - -func (filer *FilerEmbedded) CreateFile(filePath string, fid string) (err error) { - dir, file := filepath.Split(filePath) - dirId, e := filer.directories.MakeDirectory(dir) - if e != nil { - return e - } - return filer.files.CreateFile(dirId, file, fid) -} -func (filer *FilerEmbedded) FindFile(filePath string) (fid string, err error) { - dir, file := filepath.Split(filePath) - return filer.findFileEntry(dir, file) -} -func (filer *FilerEmbedded) findFileEntry(parentPath string, fileName string) (fid string, err error) { - dirId, e := filer.directories.findDirectoryId(parentPath) - if e != nil { - return "", e - } - return filer.files.FindFile(dirId, fileName) -} - -func (filer *FilerEmbedded) LookupDirectoryEntry(dirPath string, name string) (found bool, fileId string, err error) { - if _, err = filer.directories.findDirectory(filepath.Join(dirPath, name)); err == nil { - return true, "", nil - } - if fileId, err = filer.findFileEntry(dirPath, name); err == nil { - return true, fileId, nil - } - return false, "", err -} -func (filer *FilerEmbedded) ListDirectories(dirPath string) (dirs []filer.DirectoryName, err error) { - return filer.directories.ListDirectories(dirPath) -} -func (filer *FilerEmbedded) ListFiles(dirPath string, lastFileName string, limit int) (files []filer.FileEntry, err error) { - dirId, e := filer.directories.findDirectoryId(dirPath) - if e != nil { - return nil, e - } - return filer.files.ListFiles(dirId, lastFileName, limit), nil -} -func (filer *FilerEmbedded) DeleteDirectory(dirPath string, recursive bool) (err error) { - dirId, e := filer.directories.findDirectoryId(dirPath) - if e != nil { - return e - } - if sub_dirs, sub_err := filer.directories.ListDirectories(dirPath); sub_err == nil { - if len(sub_dirs) > 0 && !recursive { - return fmt.Errorf("Fail to delete directory %s: %d sub directories found!", dirPath, len(sub_dirs)) - } - for _, sub := range sub_dirs { - if delete_sub_err := filer.DeleteDirectory(filepath.Join(dirPath, string(sub)), recursive); delete_sub_err != nil { - return delete_sub_err - } - } - } - list := filer.files.ListFiles(dirId, "", 100) - if len(list) != 0 && !recursive { - if !recursive { - return fmt.Errorf("Fail to delete non-empty directory %s!", dirPath) - } - } - for { - if len(list) == 0 { - return filer.directories.DeleteDirectory(dirPath) - } - var fids []string - for _, fileEntry := range list { - fids = append(fids, string(fileEntry.Id)) - } - if result_list, delete_file_err := operation.DeleteFiles(filer.master, fids); delete_file_err != nil { - return delete_file_err - } else { - if len(result_list.Errors) > 0 { - return errors.New(strings.Join(result_list.Errors, "\n")) - } - } - lastFile := list[len(list)-1] - list = filer.files.ListFiles(dirId, lastFile.Name, 100) - } - -} - -func (filer *FilerEmbedded) DeleteFile(filePath string) (fid string, err error) { - dir, file := filepath.Split(filePath) - dirId, e := filer.directories.findDirectoryId(dir) - if e != nil { - return "", e - } - return filer.files.DeleteFile(dirId, file) -} - -/* -Move a folder or a file, with 4 Use cases: -mv fromDir toNewDir -mv fromDir toOldDir -mv fromFile toDir -mv fromFile toFile -*/ -func (filer *FilerEmbedded) Move(fromPath string, toPath string) error { - filer.mvMutex.Lock() - defer filer.mvMutex.Unlock() - - if _, dir_err := filer.directories.findDirectoryId(fromPath); dir_err == nil { - if _, err := filer.directories.findDirectoryId(toPath); err == nil { - // move folder under an existing folder - return filer.directories.MoveUnderDirectory(fromPath, toPath, "") - } - // move folder to a new folder - return filer.directories.MoveUnderDirectory(fromPath, filepath.Dir(toPath), filepath.Base(toPath)) - } - if fid, file_err := filer.DeleteFile(fromPath); file_err == nil { - if _, err := filer.directories.findDirectoryId(toPath); err == nil { - // move file under an existing folder - return filer.CreateFile(filepath.Join(toPath, filepath.Base(fromPath)), fid) - } - // move to a folder with new name - return filer.CreateFile(toPath, fid) - } - return fmt.Errorf("File %s is not found!", fromPath) -} diff --git a/weed/filer/embedded_filer/files_in_leveldb.go b/weed/filer/embedded_filer/files_in_leveldb.go deleted file mode 100644 index bde41d8a2..000000000 --- a/weed/filer/embedded_filer/files_in_leveldb.go +++ /dev/null @@ -1,87 +0,0 @@ -package embedded_filer - -import ( - "bytes" - - "github.com/chrislusf/seaweedfs/weed/filer" - "github.com/chrislusf/seaweedfs/weed/glog" - "github.com/syndtr/goleveldb/leveldb" - "github.com/syndtr/goleveldb/leveldb/util" -) - -/* -The entry in level db has this format: - key: genKey(dirId, fileName) - value: []byte(fid) -And genKey(dirId, fileName) use first 4 bytes to store dirId, and rest for fileName -*/ - -type FileListInLevelDb struct { - db *leveldb.DB -} - -func NewFileListInLevelDb(dir string) (fl *FileListInLevelDb, err error) { - fl = &FileListInLevelDb{} - if fl.db, err = leveldb.OpenFile(dir, nil); err != nil { - return - } - return -} - -func genKey(dirId DirectoryId, fileName string) []byte { - ret := make([]byte, 0, 4+len(fileName)) - for i := 3; i >= 0; i-- { - ret = append(ret, byte(dirId>>(uint(i)*8))) - } - ret = append(ret, []byte(fileName)...) - return ret -} - -func (fl *FileListInLevelDb) CreateFile(dirId DirectoryId, fileName string, fid string) (err error) { - glog.V(4).Infoln("directory", dirId, "fileName", fileName, "fid", fid) - return fl.db.Put(genKey(dirId, fileName), []byte(fid), nil) -} -func (fl *FileListInLevelDb) DeleteFile(dirId DirectoryId, fileName string) (fid string, err error) { - if fid, err = fl.FindFile(dirId, fileName); err != nil { - if err == leveldb.ErrNotFound { - return "", nil - } - return - } - err = fl.db.Delete(genKey(dirId, fileName), nil) - return fid, err -} -func (fl *FileListInLevelDb) FindFile(dirId DirectoryId, fileName string) (fid string, err error) { - data, e := fl.db.Get(genKey(dirId, fileName), nil) - if e == leveldb.ErrNotFound { - return "", filer.ErrNotFound - } else if e != nil { - return "", e - } - return string(data), nil -} -func (fl *FileListInLevelDb) ListFiles(dirId DirectoryId, lastFileName string, limit int) (files []filer.FileEntry) { - glog.V(4).Infoln("directory", dirId, "lastFileName", lastFileName, "limit", limit) - dirKey := genKey(dirId, "") - iter := fl.db.NewIterator(&util.Range{Start: genKey(dirId, lastFileName)}, nil) - limitCounter := 0 - for iter.Next() { - key := iter.Key() - if !bytes.HasPrefix(key, dirKey) { - break - } - fileName := string(key[len(dirKey):]) - if fileName == lastFileName { - continue - } - limitCounter++ - if limit > 0 { - if limitCounter > limit { - break - } - } - files = append(files, filer.FileEntry{Name: fileName, Id: filer.FileId(string(iter.Value()))}) - } - iter.Release() - return -} diff --git a/weed/filer/filer.go b/weed/filer/filer.go deleted file mode 100644 index 8f6344975..000000000 --- a/weed/filer/filer.go +++ /dev/null @@ -1,29 +0,0 @@ -package filer - -import ( - "errors" -) - -type FileId string //file id in SeaweedFS - -type FileEntry struct { - Name string `json:"name,omitempty"` //file name without path - Id FileId `json:"fid,omitempty"` -} - -type DirectoryName string - -type Filer interface { - CreateFile(fullFileName string, fid string) (err error) - FindFile(fullFileName string) (fid string, err error) - DeleteFile(fullFileName string) (fid string, err error) - - //Optional functions. embedded filer support these - ListDirectories(dirPath string) (dirs []DirectoryName, err error) - ListFiles(dirPath string, lastFileName string, limit int) (files []FileEntry, err error) - DeleteDirectory(dirPath string, recursive bool) (err error) - Move(fromPath string, toPath string) (err error) - LookupDirectoryEntry(dirPath string, name string) (found bool, fileId string, err error) -} - -var ErrNotFound = errors.New("filer: no entry is found in filer store") diff --git a/weed/filer/flat_namespace/flat_namespace_filer.go b/weed/filer/flat_namespace/flat_namespace_filer.go deleted file mode 100644 index 4173c41ee..000000000 --- a/weed/filer/flat_namespace/flat_namespace_filer.go +++ /dev/null @@ -1,66 +0,0 @@ -package flat_namespace - -import ( - "errors" - - "github.com/chrislusf/seaweedfs/weed/filer" - "path/filepath" -) - -type FlatNamespaceFiler struct { - master string - store FlatNamespaceStore -} - -var ( - ErrNotImplemented = errors.New("Not Implemented for flat namespace meta data store") -) - -func NewFlatNamespaceFiler(master string, store FlatNamespaceStore) *FlatNamespaceFiler { - return &FlatNamespaceFiler{ - master: master, - store: store, - } -} - -func (filer *FlatNamespaceFiler) CreateFile(fullFileName string, fid string) (err error) { - return filer.store.Put(fullFileName, fid) -} -func (filer *FlatNamespaceFiler) FindFile(fullFileName string) (fid string, err error) { - return filer.store.Get(fullFileName) -} -func (filer *FlatNamespaceFiler) LookupDirectoryEntry(dirPath string, name string) (found bool, fileId string, err error) { - if fileId, err = filer.FindFile(filepath.Join(dirPath, name)); err == nil { - return true, fileId, nil - } - return false, "", err -} -func (filer *FlatNamespaceFiler) ListDirectories(dirPath string) (dirs []filer.DirectoryName, err error) { - return nil, ErrNotImplemented -} -func (filer *FlatNamespaceFiler) ListFiles(dirPath string, lastFileName string, limit int) (files []filer.FileEntry, err error) { - return nil, ErrNotImplemented -} -func (filer *FlatNamespaceFiler) DeleteDirectory(dirPath string, recursive bool) (err error) { - return ErrNotImplemented -} - -func (filer *FlatNamespaceFiler) DeleteFile(fullFileName string) (fid string, err error) { - fid, err = filer.FindFile(fullFileName) - if err != nil { - return "", err - } - - err = filer.store.Delete(fullFileName) - if err != nil { - return "", err - } - - return fid, nil - //return filer.store.Delete(fullFileName) - //are you kidding me!!!! -} - -func (filer *FlatNamespaceFiler) Move(fromPath string, toPath string) error { - return ErrNotImplemented -} diff --git a/weed/filer/flat_namespace/flat_namespace_store.go b/weed/filer/flat_namespace/flat_namespace_store.go deleted file mode 100644 index dc158f7ad..000000000 --- a/weed/filer/flat_namespace/flat_namespace_store.go +++ /dev/null @@ -1,9 +0,0 @@ -package flat_namespace - -import () - -type FlatNamespaceStore interface { - Put(fullFileName string, fid string) (err error) - Get(fullFileName string) (fid string, err error) - Delete(fullFileName string) (err error) -} diff --git a/weed/filer/mysql_store/README.md b/weed/filer/mysql_store/README.md deleted file mode 100644 index 6efeb1c54..000000000 --- a/weed/filer/mysql_store/README.md +++ /dev/null @@ -1,67 +0,0 @@ -#MySQL filer mapping store - -## Schema format - - -Basically, uriPath and fid are the key elements stored in MySQL. In view of the optimization and user's usage, -adding primary key with integer type and involving createTime, updateTime, status fields should be somewhat meaningful. -Of course, you could customize the schema per your concretely circumstance freely. - -

-CREATE TABLE IF NOT EXISTS `filer_mapping` (
-  `id` bigint(20) NOT NULL AUTO_INCREMENT,
-  `uriPath` char(256) NOT NULL DEFAULT "" COMMENT 'http uriPath',
-  `fid` char(36) NOT NULL DEFAULT "" COMMENT 'seaweedfs fid',
-  `createTime` int(10) NOT NULL DEFAULT 0 COMMENT 'createdTime in unix timestamp',
-  `updateTime` int(10) NOT NULL DEFAULT 0 COMMENT 'updatedTime in unix timestamp',
-  `remark` varchar(20) NOT NULL DEFAULT "" COMMENT 'reserverd field',
-  `status` tinyint(2) DEFAULT '1' COMMENT 'resource status',
-  PRIMARY KEY (`id`),
-  UNIQUE KEY `index_uriPath` (`uriPath`)
-) DEFAULT CHARSET=utf8;
-
- - -The MySQL 's config params is not added into the weed command option as other stores(redis,cassandra). Instead, -We created a config file(json format) for them. TOML,YAML or XML also should be OK. But TOML and YAML need import thirdparty package -while XML is a little bit complex. - -The sample config file's content is below: - -

-{
-    "mysql": [
-        {
-            "User": "root",
-            "Password": "root",
-            "HostName": "127.0.0.1",
-            "Port": 3306,
-            "DataBase": "seaweedfs"
-        },
-        {
-            "User": "root",
-            "Password": "root",
-            "HostName": "127.0.0.2",
-            "Port": 3306,
-            "DataBase": "seaweedfs"
-        }
-    ],
-    "IsSharding":true,
-    "ShardCount":1024
-}
-
- - -The "mysql" field in above conf file is an array which include all mysql instances you prepared to store sharding data. - -1. If one mysql instance is enough, just keep one instance in "mysql" field. - -2. If table sharding at a specific mysql instance is needed , mark "IsSharding" field with true and specify total table sharding numbers using "ShardCount" field. - -3. If the mysql service could be auto scaled transparently in your environment, just config one mysql instance(usually it's a frondend proxy or VIP),and mark "IsSharding" with false value - -4. If you prepare more than one mysql instance and have no plan to use table sharding for any instance(mark isSharding with false), instance sharding will still be done implicitly - - - - diff --git a/weed/filer/mysql_store/mysql_store.go b/weed/filer/mysql_store/mysql_store.go deleted file mode 100644 index 4a8e889df..000000000 --- a/weed/filer/mysql_store/mysql_store.go +++ /dev/null @@ -1,270 +0,0 @@ -package mysql_store - -import ( - "database/sql" - "fmt" - "hash/crc32" - "sync" - "time" - - "github.com/chrislusf/seaweedfs/weed/filer" - - _ "github.com/go-sql-driver/mysql" -) - -const ( - sqlUrl = "%s:%s@tcp(%s:%d)/%s?charset=utf8" - default_maxIdleConnections = 100 - default_maxOpenConnections = 50 - default_maxTableNums = 1024 - tableName = "filer_mapping" -) - -var ( - _init_db sync.Once - _db_connections []*sql.DB -) - -type MySqlConf struct { - User string - Password string - HostName string - Port int - DataBase string - MaxIdleConnections int - MaxOpenConnections int -} - -type ShardingConf struct { - IsSharding bool `json:"isSharding"` - ShardCount int `json:"shardCount"` -} - -type MySqlStore struct { - dbs []*sql.DB - isSharding bool - shardCount int -} - -func getDbConnection(confs []MySqlConf) []*sql.DB { - _init_db.Do(func() { - for _, conf := range confs { - - sqlUrl := fmt.Sprintf(sqlUrl, conf.User, conf.Password, conf.HostName, conf.Port, conf.DataBase) - var dbErr error - _db_connection, dbErr := sql.Open("mysql", sqlUrl) - if dbErr != nil { - _db_connection.Close() - _db_connection = nil - panic(dbErr) - } - var maxIdleConnections, maxOpenConnections int - - if conf.MaxIdleConnections != 0 { - maxIdleConnections = conf.MaxIdleConnections - } else { - maxIdleConnections = default_maxIdleConnections - } - if conf.MaxOpenConnections != 0 { - maxOpenConnections = conf.MaxOpenConnections - } else { - maxOpenConnections = default_maxOpenConnections - } - - _db_connection.SetMaxIdleConns(maxIdleConnections) - _db_connection.SetMaxOpenConns(maxOpenConnections) - _db_connections = append(_db_connections, _db_connection) - } - }) - return _db_connections -} - -func NewMysqlStore(confs []MySqlConf, isSharding bool, shardCount int) *MySqlStore { - ms := &MySqlStore{ - dbs: getDbConnection(confs), - isSharding: isSharding, - shardCount: shardCount, - } - - for _, db := range ms.dbs { - if !isSharding { - ms.shardCount = 1 - } else { - if ms.shardCount == 0 { - ms.shardCount = default_maxTableNums - } - } - for i := 0; i < ms.shardCount; i++ { - if err := ms.createTables(db, tableName, i); err != nil { - fmt.Printf("create table failed %v", err) - } - } - } - - return ms -} - -func (s *MySqlStore) hash(fullFileName string) (instance_offset, table_postfix int) { - hash_value := crc32.ChecksumIEEE([]byte(fullFileName)) - instance_offset = int(hash_value) % len(s.dbs) - table_postfix = int(hash_value) % s.shardCount - return -} - -func (s *MySqlStore) parseFilerMappingInfo(path string) (instanceId int, tableFullName string, err error) { - instance_offset, table_postfix := s.hash(path) - instanceId = instance_offset - if s.isSharding { - tableFullName = fmt.Sprintf("%s_%04d", tableName, table_postfix) - } else { - tableFullName = tableName - } - return -} - -func (s *MySqlStore) Get(fullFilePath string) (fid string, err error) { - instance_offset, tableFullName, err := s.parseFilerMappingInfo(fullFilePath) - if err != nil { - return "", fmt.Errorf("MySqlStore Get operation can not parse file path %s: err is %v", fullFilePath, err) - } - fid, err = s.query(fullFilePath, s.dbs[instance_offset], tableFullName) - if err == sql.ErrNoRows { - //Could not found - err = filer.ErrNotFound - } - return fid, err -} - -func (s *MySqlStore) Put(fullFilePath string, fid string) (err error) { - var tableFullName string - - instance_offset, tableFullName, err := s.parseFilerMappingInfo(fullFilePath) - if err != nil { - return fmt.Errorf("MySqlStore Put operation can not parse file path %s: err is %v", fullFilePath, err) - } - var old_fid string - if old_fid, err = s.query(fullFilePath, s.dbs[instance_offset], tableFullName); err != nil && err != sql.ErrNoRows { - return fmt.Errorf("MySqlStore Put operation failed when querying path %s: err is %v", fullFilePath, err) - } else { - if len(old_fid) == 0 { - err = s.insert(fullFilePath, fid, s.dbs[instance_offset], tableFullName) - err = fmt.Errorf("MySqlStore Put operation failed when inserting path %s with fid %s : err is %v", fullFilePath, fid, err) - } else { - err = s.update(fullFilePath, fid, s.dbs[instance_offset], tableFullName) - err = fmt.Errorf("MySqlStore Put operation failed when updating path %s with fid %s : err is %v", fullFilePath, fid, err) - } - } - return -} - -func (s *MySqlStore) Delete(fullFilePath string) (err error) { - var fid string - instance_offset, tableFullName, err := s.parseFilerMappingInfo(fullFilePath) - if err != nil { - return fmt.Errorf("MySqlStore Delete operation can not parse file path %s: err is %v", fullFilePath, err) - } - if fid, err = s.query(fullFilePath, s.dbs[instance_offset], tableFullName); err != nil { - return fmt.Errorf("MySqlStore Delete operation failed when querying path %s: err is %v", fullFilePath, err) - } else if fid == "" { - return nil - } - if err = s.delete(fullFilePath, s.dbs[instance_offset], tableFullName); err != nil { - return fmt.Errorf("MySqlStore Delete operation failed when deleting path %s: err is %v", fullFilePath, err) - } else { - return nil - } -} - -func (s *MySqlStore) Close() { - for _, db := range s.dbs { - db.Close() - } -} - -var createTable = ` -CREATE TABLE IF NOT EXISTS %s ( - id bigint(20) NOT NULL AUTO_INCREMENT, - uriPath char(255) NOT NULL DEFAULT "" COMMENT 'http uriPath', - fid char(36) NOT NULL DEFAULT "" COMMENT 'seaweedfs fid', - createTime int(10) NOT NULL DEFAULT 0 COMMENT 'createdTime in unix timestamp', - updateTime int(10) NOT NULL DEFAULT 0 COMMENT 'updatedTime in unix timestamp', - remark varchar(20) NOT NULL DEFAULT "" COMMENT 'reserverd field', - status tinyint(2) DEFAULT '1' COMMENT 'resource status', - PRIMARY KEY (id), - UNIQUE KEY index_uriPath (uriPath) -) DEFAULT CHARSET=utf8; -` - -func (s *MySqlStore) createTables(db *sql.DB, tableName string, postfix int) error { - var realTableName string - if s.isSharding { - realTableName = fmt.Sprintf("%s_%4d", tableName, postfix) - } else { - realTableName = tableName - } - - stmt, err := db.Prepare(fmt.Sprintf(createTable, realTableName)) - if err != nil { - return err - } - defer stmt.Close() - - _, err = stmt.Exec() - if err != nil { - return err - } - return nil -} - -func (s *MySqlStore) query(uriPath string, db *sql.DB, tableName string) (string, error) { - sqlStatement := "SELECT fid FROM %s WHERE uriPath=?" - row := db.QueryRow(fmt.Sprintf(sqlStatement, tableName), uriPath) - var fid string - err := row.Scan(&fid) - if err != nil { - return "", err - } - return fid, nil -} - -func (s *MySqlStore) update(uriPath string, fid string, db *sql.DB, tableName string) error { - sqlStatement := "UPDATE %s SET fid=?, updateTime=? WHERE uriPath=?" - res, err := db.Exec(fmt.Sprintf(sqlStatement, tableName), fid, time.Now().Unix(), uriPath) - if err != nil { - return err - } - - _, err = res.RowsAffected() - if err != nil { - return err - } - return nil -} - -func (s *MySqlStore) insert(uriPath string, fid string, db *sql.DB, tableName string) error { - sqlStatement := "INSERT INTO %s (uriPath,fid,createTime) VALUES(?,?,?)" - res, err := db.Exec(fmt.Sprintf(sqlStatement, tableName), uriPath, fid, time.Now().Unix()) - if err != nil { - return err - } - - _, err = res.RowsAffected() - if err != nil { - return err - } - return nil -} - -func (s *MySqlStore) delete(uriPath string, db *sql.DB, tableName string) error { - sqlStatement := "DELETE FROM %s WHERE uriPath=?" - res, err := db.Exec(fmt.Sprintf(sqlStatement, tableName), uriPath) - if err != nil { - return err - } - - _, err = res.RowsAffected() - if err != nil { - return err - } - return nil -} diff --git a/weed/filer/mysql_store/mysql_store_test.go b/weed/filer/mysql_store/mysql_store_test.go deleted file mode 100644 index 1c9765c59..000000000 --- a/weed/filer/mysql_store/mysql_store_test.go +++ /dev/null @@ -1,30 +0,0 @@ -package mysql_store - -import ( - "encoding/json" - "hash/crc32" - "testing" -) - -func TestGenerateMysqlConf(t *testing.T) { - var conf []MySqlConf - conf = append(conf, MySqlConf{ - User: "root", - Password: "root", - HostName: "localhost", - Port: 3306, - DataBase: "seaweedfs", - }) - body, err := json.Marshal(conf) - if err != nil { - t.Errorf("json encoding err %s", err.Error()) - } - t.Logf("json output is %s", string(body)) -} - -func TestCRC32FullPathName(t *testing.T) { - fullPathName := "/prod-bucket/law632191483895612493300-signed.pdf" - hash_value := crc32.ChecksumIEEE([]byte(fullPathName)) - table_postfix := int(hash_value) % 1024 - t.Logf("table postfix %d", table_postfix) -} diff --git a/weed/filer/postgres_store/postgres_native.go b/weed/filer/postgres_store/postgres_native.go deleted file mode 100644 index 61bd4210c..000000000 --- a/weed/filer/postgres_store/postgres_native.go +++ /dev/null @@ -1,456 +0,0 @@ -package postgres_store - -import ( - "database/sql" - "fmt" - "path/filepath" - "time" - - "github.com/chrislusf/seaweedfs/weed/filer" - "github.com/chrislusf/seaweedfs/weed/glog" - - _ "github.com/lib/pq" - _ "path/filepath" - "strings" -) - -type DirectoryId int32 - -func databaseExists(db *sql.DB, databaseName string) (bool, error) { - sqlStatement := "SELECT datname from pg_database WHERE datname='%s'" - row := db.QueryRow(fmt.Sprintf(sqlStatement, databaseName)) - - var dbName string - err := row.Scan(&dbName) - if err != nil { - if err == sql.ErrNoRows { - return false, nil - } - return false, err - } - return true, nil -} - -func createDatabase(db *sql.DB, databaseName string) error { - sqlStatement := "CREATE DATABASE %s ENCODING='UTF8'" - _, err := db.Exec(fmt.Sprintf(sqlStatement, databaseName)) - return err -} - -func getDbConnection(conf PostgresConf) *sql.DB { - _init_db.Do(func() { - - sqlUrl := fmt.Sprintf("host=%s port=%d user=%s password=%s dbname=%s sslmode=%s connect_timeout=30", conf.HostName, conf.Port, conf.User, conf.Password, "postgres", conf.SslMode) - glog.V(3).Infoln("Opening postgres master database") - - var dbErr error - _db_connection, dbErr := sql.Open("postgres", sqlUrl) - if dbErr != nil { - _db_connection.Close() - _db_connection = nil - panic(dbErr) - } - - pingErr := _db_connection.Ping() - if pingErr != nil { - _db_connection.Close() - _db_connection = nil - panic(pingErr) - } - - glog.V(3).Infoln("Checking to see if DB exists: ", conf.DataBase) - var existsErr error - dbExists, existsErr := databaseExists(_db_connection, conf.DataBase) - if existsErr != nil { - _db_connection.Close() - _db_connection = nil - panic(existsErr) - } - - if !dbExists { - glog.V(3).Infoln("Database doesn't exist. Attempting to create one: ", conf.DataBase) - createErr := createDatabase(_db_connection, conf.DataBase) - if createErr != nil { - _db_connection.Close() - _db_connection = nil - panic(createErr) - } - } - - glog.V(3).Infoln("Closing master postgres database and opening configured database: ", conf.DataBase) - _db_connection.Close() - _db_connection = nil - - sqlUrl = fmt.Sprintf("host=%s port=%d user=%s password=%s dbname=%s sslmode=%s connect_timeout=30", conf.HostName, conf.Port, conf.User, conf.Password, conf.DataBase, conf.SslMode) - _db_connection, dbErr = sql.Open("postgres", sqlUrl) - if dbErr != nil { - _db_connection.Close() - _db_connection = nil - panic(dbErr) - } - - pingErr = _db_connection.Ping() - if pingErr != nil { - _db_connection.Close() - _db_connection = nil - panic(pingErr) - } - - maxIdleConnections, maxOpenConnections := default_maxIdleConnections, default_maxOpenConnections - if conf.MaxIdleConnections != 0 { - maxIdleConnections = conf.MaxIdleConnections - } - if conf.MaxOpenConnections != 0 { - maxOpenConnections = conf.MaxOpenConnections - } - - _db_connection.SetMaxIdleConns(maxIdleConnections) - _db_connection.SetMaxOpenConns(maxOpenConnections) - }) - return _db_connection -} - -var createDirectoryTable = ` - -CREATE TABLE IF NOT EXISTS %s ( - id BIGSERIAL NOT NULL, - directoryRoot VARCHAR(1024) NOT NULL DEFAULT '', - directoryName VARCHAR(1024) NOT NULL DEFAULT '', - CONSTRAINT unique_directory UNIQUE (directoryRoot, directoryName) -); -` - -var createFileTable = ` - -CREATE TABLE IF NOT EXISTS %s ( - id BIGSERIAL NOT NULL, - directoryPart VARCHAR(1024) NOT NULL DEFAULT '', - filePart VARCHAR(1024) NOT NULL DEFAULT '', - fid VARCHAR(36) NOT NULL DEFAULT '', - createTime BIGINT NOT NULL DEFAULT 0, - updateTime BIGINT NOT NULL DEFAULT 0, - remark VARCHAR(20) NOT NULL DEFAULT '', - status SMALLINT NOT NULL DEFAULT '1', - PRIMARY KEY (id), - CONSTRAINT %s_unique_file UNIQUE (directoryPart, filePart) -); -` - -func (s *PostgresStore) createDirectoriesTable() error { - glog.V(3).Infoln("Creating postgres table if it doesn't exist: ", directoriesTableName) - - sqlCreate := fmt.Sprintf(createDirectoryTable, directoriesTableName) - - stmt, err := s.db.Prepare(sqlCreate) - if err != nil { - return err - } - defer stmt.Close() - - _, err = stmt.Exec() - if err != nil { - return err - } - return nil -} - -func (s *PostgresStore) createFilesTable() error { - - glog.V(3).Infoln("Creating postgres table if it doesn't exist: ", filesTableName) - - sqlCreate := fmt.Sprintf(createFileTable, filesTableName, filesTableName) - - stmt, err := s.db.Prepare(sqlCreate) - if err != nil { - return err - } - defer stmt.Close() - - _, err = stmt.Exec() - if err != nil { - return err - } - return nil -} - -func (s *PostgresStore) query(uriPath string) (string, error) { - directoryPart, filePart := filepath.Split(uriPath) - sqlStatement := fmt.Sprintf("SELECT fid FROM %s WHERE directoryPart=$1 AND filePart=$2", filesTableName) - - row := s.db.QueryRow(sqlStatement, directoryPart, filePart) - var fid string - err := row.Scan(&fid) - - glog.V(3).Infof("Postgres query -- looking up path '%s' and found id '%s' ", uriPath, fid) - - if err != nil { - return "", err - } - return fid, nil -} - -func (s *PostgresStore) update(uriPath string, fid string) error { - directoryPart, filePart := filepath.Split(uriPath) - sqlStatement := fmt.Sprintf("UPDATE %s SET fid=$1, updateTime=$2 WHERE directoryPart=$3 AND filePart=$4", filesTableName) - - glog.V(3).Infof("Postgres query -- updating path '%s' with id '%s'", uriPath, fid) - - res, err := s.db.Exec(sqlStatement, fid, time.Now().Unix(), directoryPart, filePart) - if err != nil { - return err - } - - _, err = res.RowsAffected() - if err != nil { - return err - } - return nil -} - -func (s *PostgresStore) insert(uriPath string, fid string) error { - directoryPart, filePart := filepath.Split(uriPath) - - existingId, _, _ := s.lookupDirectory(directoryPart) - if existingId == 0 { - s.recursiveInsertDirectory(directoryPart) - } - - sqlStatement := fmt.Sprintf("INSERT INTO %s (directoryPart,filePart,fid,createTime) VALUES($1, $2, $3, $4)", filesTableName) - glog.V(3).Infof("Postgres query -- inserting path '%s' with id '%s'", uriPath, fid) - - res, err := s.db.Exec(sqlStatement, directoryPart, filePart, fid, time.Now().Unix()) - - if err != nil { - return err - } - - rows, err := res.RowsAffected() - if rows != 1 { - return fmt.Errorf("Postgres insert -- rows affected = %d. Expecting 1", rows) - } - if err != nil { - return err - } - - return nil -} - -func (s *PostgresStore) recursiveInsertDirectory(dirPath string) { - pathParts := strings.Split(dirPath, "/") - - var workingPath string = "/" - for _, part := range pathParts { - if part == "" { - continue - } - workingPath += (part + "/") - existingId, _, _ := s.lookupDirectory(workingPath) - if existingId == 0 { - s.insertDirectory(workingPath) - } - } -} - -func (s *PostgresStore) insertDirectory(dirPath string) { - pathParts := strings.Split(dirPath, "/") - - directoryRoot := "/" - directoryName := "" - if len(pathParts) > 1 { - directoryRoot = strings.Join(pathParts[0:len(pathParts)-2], "/") + "/" - directoryName = strings.Join(pathParts[len(pathParts)-2:], "/") - } else if len(pathParts) == 1 { - directoryRoot = "/" - directoryName = pathParts[0] + "/" - } - sqlInsertDirectoryStatement := fmt.Sprintf("INSERT INTO %s (directoryroot, directoryname) "+ - "SELECT $1, $2 WHERE NOT EXISTS ( SELECT id FROM %s WHERE directoryroot=$3 AND directoryname=$4 )", - directoriesTableName, directoriesTableName) - - glog.V(4).Infof("Postgres query -- Inserting directory (if it doesn't exist) - root = %s, name = %s", - directoryRoot, directoryName) - - _, err := s.db.Exec(sqlInsertDirectoryStatement, directoryRoot, directoryName, directoryRoot, directoryName) - if err != nil { - glog.V(0).Infof("Postgres query -- Error inserting directory - root = %s, name = %s: %s", - directoryRoot, directoryName, err) - } -} - -func (s *PostgresStore) delete(uriPath string) error { - directoryPart, filePart := filepath.Split(uriPath) - sqlStatement := fmt.Sprintf("DELETE FROM %s WHERE directoryPart=$1 AND filePart=$2", filesTableName) - - glog.V(3).Infof("Postgres query -- deleting path '%s'", uriPath) - - res, err := s.db.Exec(sqlStatement, directoryPart, filePart) - if err != nil { - return err - } - - _, err = res.RowsAffected() - if err != nil { - return err - } - return nil -} - -func (s *PostgresStore) lookupDirectory(dirPath string) (DirectoryId, string, error) { - directoryRoot, directoryName := s.mySplitPath(dirPath) - - sqlStatement := fmt.Sprintf("SELECT id, directoryroot, directoryname FROM %s WHERE directoryRoot=$1 AND directoryName=$2", directoriesTableName) - - row := s.db.QueryRow(sqlStatement, directoryRoot, directoryName) - var id DirectoryId - var dirRoot string - var dirName string - err := row.Scan(&id, &dirRoot, &dirName) - - glog.V(3).Infof("Postgres lookupDirectory -- looking up directory '%s' and found id '%d', root '%s', name '%s' ", dirPath, id, dirRoot, dirName) - - if err != nil { - return 0, "", err - } - return id, filepath.Join(dirRoot, dirName), err -} - -func (s *PostgresStore) findDirectories(dirPath string, limit int) (dirs []filer.DirectoryName, err error) { - sqlStatement := fmt.Sprintf("SELECT id, directoryroot, directoryname FROM %s WHERE directoryRoot=$1 AND directoryName != '' ORDER BY id LIMIT $2", directoriesTableName) - rows, err := s.db.Query(sqlStatement, dirPath, limit) - - if err != nil { - glog.V(0).Infof("Postgres findDirectories error: %s", err) - } - - if rows != nil { - defer rows.Close() - for rows.Next() { - var id DirectoryId - var directoryRoot string - var directoryName string - - scanErr := rows.Scan(&id, &directoryRoot, &directoryName) - if scanErr != nil { - err = scanErr - } - dirs = append(dirs, filer.DirectoryName(directoryName)) - } - } - return -} - -func (s *PostgresStore) safeToDeleteDirectory(dirPath string, recursive bool) bool { - if recursive { - return true - } - sqlStatement := fmt.Sprintf("SELECT id FROM %s WHERE directoryRoot LIKE $1 LIMIT 1", directoriesTableName) - row := s.db.QueryRow(sqlStatement, dirPath+"%") - - var id DirectoryId - err := row.Scan(&id) - if err != nil { - if err == sql.ErrNoRows { - return true - } - } - return false -} - -func (s *PostgresStore) mySplitPath(dirPath string) (directoryRoot string, directoryName string) { - pathParts := strings.Split(dirPath, "/") - directoryRoot = "/" - directoryName = "" - if len(pathParts) > 1 { - directoryRoot = strings.Join(pathParts[0:len(pathParts)-2], "/") + "/" - directoryName = strings.Join(pathParts[len(pathParts)-2:], "/") - } else if len(pathParts) == 1 { - directoryRoot = "/" - directoryName = pathParts[0] + "/" - } - return directoryRoot, directoryName -} - -func (s *PostgresStore) deleteDirectory(dirPath string, recursive bool) (err error) { - directoryRoot, directoryName := s.mySplitPath(dirPath) - - // delete files - sqlStatement := fmt.Sprintf("DELETE FROM %s WHERE directorypart=$1", filesTableName) - _, err = s.db.Exec(sqlStatement, dirPath) - if err != nil { - return err - } - - // delete specific directory if it is empty or recursive delete was requested - safeToDelete := s.safeToDeleteDirectory(dirPath, recursive) - if safeToDelete { - sqlStatement = fmt.Sprintf("DELETE FROM %s WHERE directoryRoot=$1 AND directoryName=$2", directoriesTableName) - _, err = s.db.Exec(sqlStatement, directoryRoot, directoryName) - if err != nil { - return err - } - } - - if recursive { - // delete descendant files - sqlStatement = fmt.Sprintf("DELETE FROM %s WHERE directorypart LIKE $1", filesTableName) - _, err = s.db.Exec(sqlStatement, dirPath+"%") - if err != nil { - return err - } - - // delete descendant directories - sqlStatement = fmt.Sprintf("DELETE FROM %s WHERE directoryRoot LIKE $1", directoriesTableName) - _, err = s.db.Exec(sqlStatement, dirPath+"%") - if err != nil { - return err - } - } - - return err -} - -func (s *PostgresStore) findFiles(dirPath string, lastFileName string, limit int) (files []filer.FileEntry, err error) { - var rows *sql.Rows = nil - - if lastFileName == "" { - sqlStatement := - fmt.Sprintf("SELECT fid, directorypart, filepart FROM %s WHERE directorypart=$1 ORDER BY id LIMIT $2", filesTableName) - rows, err = s.db.Query(sqlStatement, dirPath, limit) - } else { - sqlStatement := - fmt.Sprintf("SELECT fid, directorypart, filepart FROM %s WHERE directorypart=$1 "+ - "AND id > (SELECT id FROM %s WHERE directoryPart=$2 AND filepart=$3) ORDER BY id LIMIT $4", - filesTableName, filesTableName) - _, lastFileNameName := filepath.Split(lastFileName) - rows, err = s.db.Query(sqlStatement, dirPath, dirPath, lastFileNameName, limit) - } - - if err != nil { - glog.V(0).Infof("Postgres find files error: %s", err) - } - - if rows != nil { - defer rows.Close() - - for rows.Next() { - var fid filer.FileId - var directoryPart string - var filePart string - - scanErr := rows.Scan(&fid, &directoryPart, &filePart) - if scanErr != nil { - err = scanErr - } - - files = append(files, filer.FileEntry{Name: filepath.Join(directoryPart, filePart), Id: fid}) - if len(files) >= limit { - break - } - } - } - - glog.V(3).Infof("Postgres findFiles -- looking up files under '%s' and found %d files. Limit=%d, lastFileName=%s", - dirPath, len(files), limit, lastFileName) - - return files, err -} diff --git a/weed/filer/postgres_store/postgres_store.go b/weed/filer/postgres_store/postgres_store.go deleted file mode 100644 index 854793f6a..000000000 --- a/weed/filer/postgres_store/postgres_store.go +++ /dev/null @@ -1,149 +0,0 @@ -package postgres_store - -import ( - "database/sql" - "errors" - "fmt" - "sync" - "github.com/chrislusf/seaweedfs/weed/filer" - "github.com/chrislusf/seaweedfs/weed/glog" - - _ "github.com/lib/pq" - _ "path/filepath" - "path/filepath" -) - -const ( - default_maxIdleConnections = 100 - default_maxOpenConnections = 50 - filesTableName = "files" - directoriesTableName = "directories" -) - -var ( - _init_db sync.Once - _db_connection *sql.DB -) - -type PostgresConf struct { - User string - Password string - HostName string - Port int - DataBase string - SslMode string - MaxIdleConnections int - MaxOpenConnections int -} - -type PostgresStore struct { - db *sql.DB - server string - user string - password string -} - -func (s *PostgresStore) CreateFile(fullFilePath string, fid string) (err error) { - - var old_fid string - if old_fid, err = s.query(fullFilePath); err != nil && err != sql.ErrNoRows { - return fmt.Errorf("PostgresStore Put operation failed when querying path %s: err is %v", fullFilePath, err) - } else { - if len(old_fid) == 0 { - err = s.insert(fullFilePath, fid) - if err != nil { - return fmt.Errorf("PostgresStore Put operation failed when inserting path %s with fid %s : err is %v", fullFilePath, fid, err) - } - } else { - err = s.update(fullFilePath, fid) - if err != nil { - return fmt.Errorf("PostgresStore Put operation failed when updating path %s with fid %s : err is %v", fullFilePath, fid, err) - } - } - } - return - -} - -func (s *PostgresStore) FindFile(fullFilePath string) (fid string, err error) { - - if err != nil { - return "", fmt.Errorf("PostgresStore Get operation can not parse file path %s: err is %v", fullFilePath, err) - } - fid, err = s.query(fullFilePath) - - return fid, err -} - -func (s *PostgresStore) LookupDirectoryEntry(dirPath string, name string) (found bool, fileId string, err error) { - fullPath := filepath.Join(dirPath, name) - if fileId, err = s.FindFile(fullPath); err == nil { - return true, fileId, nil - } - if _, _, err := s.lookupDirectory(fullPath); err == nil { - return true, "", err - } - return false, "", err -} - -func (s *PostgresStore) DeleteFile(fullFilePath string) (fid string, err error) { - if err != nil { - return "", fmt.Errorf("PostgresStore Delete operation can not parse file path %s: err is %v", fullFilePath, err) - } - if fid, err = s.query(fullFilePath); err != nil { - return "", fmt.Errorf("PostgresStore Delete operation failed when querying path %s: err is %v", fullFilePath, err) - } else if fid == "" { - return "", nil - } - if err = s.delete(fullFilePath); err != nil { - return "", fmt.Errorf("PostgresStore Delete operation failed when deleting path %s: err is %v", fullFilePath, err) - } else { - return "", nil - } -} - -func (s *PostgresStore) ListDirectories(dirPath string) (dirs []filer.DirectoryName, err error) { - - dirs, err = s.findDirectories(dirPath, 1000) - - glog.V(3).Infof("Postgres ListDirs = found %d directories under %s", len(dirs), dirPath) - - return dirs, err -} - -func (s *PostgresStore) ListFiles(dirPath string, lastFileName string, limit int) (files []filer.FileEntry, err error) { - files, err = s.findFiles(dirPath, lastFileName, limit) - return files, err -} - -func (s *PostgresStore) DeleteDirectory(dirPath string, recursive bool) (err error) { - err = s.deleteDirectory(dirPath, recursive) - if err != nil { - glog.V(0).Infof("Error in Postgres DeleteDir '%s' (recursive = '%t'): %s", err) - } - return err -} - -func (s *PostgresStore) Move(fromPath string, toPath string) (err error) { - glog.V(3).Infoln("Calling posgres_store Move") - return errors.New("Move is not yet implemented for the PostgreSQL store.") -} - -//func NewPostgresStore(master string, confs []PostgresConf, isSharding bool, shardCount int) *PostgresStore { -func NewPostgresStore(master string, conf PostgresConf) *PostgresStore { - pg := &PostgresStore{ - db: getDbConnection(conf), - } - - pg.createDirectoriesTable() - - if err := pg.createFilesTable(); err != nil { - fmt.Printf("create table failed %v", err) - } - - return pg -} - -func (s *PostgresStore) Close() { - s.db.Close() -} diff --git a/weed/filer/redis_store/redis_store.go b/weed/filer/redis_store/redis_store.go deleted file mode 100644 index 5b8362983..000000000 --- a/weed/filer/redis_store/redis_store.go +++ /dev/null @@ -1,50 +0,0 @@ -package redis_store - -import ( - "github.com/chrislusf/seaweedfs/weed/filer" - - "github.com/go-redis/redis" -) - -type RedisStore struct { - Client *redis.Client -} - -func NewRedisStore(hostPort string, password string, database int) *RedisStore { - client := redis.NewClient(&redis.Options{ - Addr: hostPort, - Password: password, - DB: database, - }) - return &RedisStore{Client: client} -} - -func (s *RedisStore) Get(fullFileName string) (fid string, err error) { - fid, err = s.Client.Get(fullFileName).Result() - if err == redis.Nil { - err = filer.ErrNotFound - } - return fid, err -} -func (s *RedisStore) Put(fullFileName string, fid string) (err error) { - _, err = s.Client.Set(fullFileName, fid, 0).Result() - if err == redis.Nil { - err = nil - } - return err -} - -// Currently the fid is not returned -func (s *RedisStore) Delete(fullFileName string) (err error) { - _, err = s.Client.Del(fullFileName).Result() - if err == redis.Nil { - err = nil - } - return err -} - -func (s *RedisStore) Close() { - if s.Client != nil { - s.Client.Close() - } -} diff --git a/weed/filer/vasto_store/design.txt b/weed/filer/vasto_store/design.txt deleted file mode 100644 index 99faa29d6..000000000 --- a/weed/filer/vasto_store/design.txt +++ /dev/null @@ -1,45 +0,0 @@ -There are two main components of a filer: directories and files. - -My previous approach was to use some sequance number to generate directoryId. -However, this is not scalable. The id generation itself is a bottleneck. -It needs careful locking and deduplication checking to get a directoryId. - -In a second design, each directory is deterministically mapped to UUID version 3, -which uses MD5 to map a tuple of to a version 3 UUID. -However, this UUID3 approach is logically the same as storing the full path. - -Storing the full path is the simplest design. - -separator is a special byte, 0x00. - -When writing a file: - => fildId, file properties -For folders: - The filer breaks the directory path into folders. - for each folder: - if it is not in cache: - check whether the folder is created in the KVS, if not: - set => directory properties - if no permission for the folder: - break - - -The filer caches the most recently used folder permissions with a TTL. - So any folder permission change needs to wait TTL interval to take effect. - - - -When listing the directory: - prefix scan of using (the folder full path + separator) as the prefix - -The downside: - 1. Rename a folder will need to recursively process all sub folders and files. - 2. Move a folder will need to recursively process all sub folders and files. -So these operations are not allowed if the folder is not empty. - -Allowing: - 1. Rename a file - 2. Move a file to a different folder - 3. Delete an empty folder - - diff --git a/weed/server/filer_server_handlers_write.go b/weed/server/filer_server_handlers_write.go index a24995956..9d5c620de 100644 --- a/weed/server/filer_server_handlers_write.go +++ b/weed/server/filer_server_handlers_write.go @@ -319,7 +319,7 @@ func (fs *FilerServer) PostHandler(w http.ResponseWriter, r *http.Request) { if entry, err := fs.filer.FindEntry(filer2.FullPath(path)); err == nil { oldFid := entry.Chunks[0].FileId operation.DeleteFile(fs.getMasterNode(), oldFid, fs.jwt(oldFid)) - } else if err != nil && err != filer.ErrNotFound { + } else if err != nil && err != filer2.ErrNotFound { glog.V(0).Infof("error %v occur when finding %s in filer store", err, path) } } From 458ada173ed6df2933c35fe0d3e3dd9457d3a975 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sun, 27 May 2018 11:52:26 -0700 Subject: [PATCH 54/83] go fmt --- weed/command/filer.go | 6 ++-- weed/command/filer_copy.go | 2 +- weed/command/mount_std.go | 2 +- .../filer2/abstract_sql/abstract_sql_store.go | 2 +- weed/filer2/cassandra/cassandra_store.go | 2 +- weed/filer2/configuration.go | 20 ++++++------- weed/filer2/entry.go | 2 +- weed/filer2/entry_codec.go | 4 +-- weed/filer2/filechunks.go | 2 +- weed/filer2/filechunks_test.go | 30 +++++++++---------- weed/filer2/filer.go | 26 ++++++++-------- weed/filer2/filerstore.go | 4 +-- weed/filer2/fullpath.go | 2 +- weed/filer2/leveldb/leveldb_store.go | 10 +++---- weed/filer2/leveldb/leveldb_store_test.go | 2 +- weed/filer2/memdb/memdb_store.go | 4 +-- weed/filer2/memdb/memdb_store_test.go | 2 +- weed/filer2/mysql/mysql_store.go | 6 ++-- weed/filer2/postgres/postgres_store.go | 6 ++-- weed/filer2/redis/redis_store.go | 2 +- weed/filesys/dir.go | 4 +-- weed/filesys/file.go | 6 ++-- weed/filesys/filehandle.go | 13 ++++---- weed/filesys/wfs.go | 4 +-- weed/images/favicon.go | 2 +- weed/server/filer_grpc_server.go | 8 ++--- weed/server/filer_server.go | 8 ++--- weed/server/filer_server_handlers_admin.go | 4 +-- weed/server/filer_server_handlers_read.go | 2 +- weed/server/filer_server_handlers_write.go | 5 ++-- weed/server/filer_ui/breadcrumb.go | 2 +- weed/server/raft_server.go | 2 +- weed/server/volume_server_handlers_admin.go | 2 +- weed/server/volume_server_handlers_ui.go | 2 +- weed/storage/needle_map_memory.go | 2 +- weed/storage/volume_checking.go | 2 +- 36 files changed, 101 insertions(+), 103 deletions(-) diff --git a/weed/command/filer.go b/weed/command/filer.go index 5f0ceab48..fd9419772 100644 --- a/weed/command/filer.go +++ b/weed/command/filer.go @@ -5,14 +5,14 @@ import ( "strconv" "time" + "github.com/chrislusf/seaweedfs/weed/filer2" "github.com/chrislusf/seaweedfs/weed/glog" + "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "github.com/chrislusf/seaweedfs/weed/server" "github.com/chrislusf/seaweedfs/weed/util" "github.com/soheilhy/cmux" - "google.golang.org/grpc/reflection" "google.golang.org/grpc" - "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" - "github.com/chrislusf/seaweedfs/weed/filer2" + "google.golang.org/grpc/reflection" ) var ( diff --git a/weed/command/filer_copy.go b/weed/command/filer_copy.go index 4c203fc94..8a754bb55 100644 --- a/weed/command/filer_copy.go +++ b/weed/command/filer_copy.go @@ -68,7 +68,7 @@ func runCopy(cmd *Command, args []string) bool { return false } filerDestination := args[len(args)-1] - fileOrDirs := args[0: len(args)-1] + fileOrDirs := args[0 : len(args)-1] filerUrl, err := url.Parse(filerDestination) if err != nil { diff --git a/weed/command/mount_std.go b/weed/command/mount_std.go index 4908bdbff..b3f038cc2 100644 --- a/weed/command/mount_std.go +++ b/weed/command/mount_std.go @@ -8,9 +8,9 @@ import ( "bazil.org/fuse" "bazil.org/fuse/fs" + "github.com/chrislusf/seaweedfs/weed/filesys" "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/util" - "github.com/chrislusf/seaweedfs/weed/filesys" ) func runMount(cmd *Command, args []string) bool { diff --git a/weed/filer2/abstract_sql/abstract_sql_store.go b/weed/filer2/abstract_sql/abstract_sql_store.go index e924fa16a..aa3b82c5b 100644 --- a/weed/filer2/abstract_sql/abstract_sql_store.go +++ b/weed/filer2/abstract_sql/abstract_sql_store.go @@ -1,8 +1,8 @@ package abstract_sql import ( - "fmt" "database/sql" + "fmt" "github.com/chrislusf/seaweedfs/weed/filer2" "github.com/chrislusf/seaweedfs/weed/glog" diff --git a/weed/filer2/cassandra/cassandra_store.go b/weed/filer2/cassandra/cassandra_store.go index 316b6cce0..a1e63b87d 100644 --- a/weed/filer2/cassandra/cassandra_store.go +++ b/weed/filer2/cassandra/cassandra_store.go @@ -2,9 +2,9 @@ package cassandra import ( "fmt" - "github.com/gocql/gocql" "github.com/chrislusf/seaweedfs/weed/filer2" "github.com/chrislusf/seaweedfs/weed/glog" + "github.com/gocql/gocql" "github.com/spf13/viper" ) diff --git a/weed/filer2/configuration.go b/weed/filer2/configuration.go index 16c0f3fd0..46ed9e056 100644 --- a/weed/filer2/configuration.go +++ b/weed/filer2/configuration.go @@ -3,8 +3,8 @@ package filer2 import ( "os" - "github.com/spf13/viper" "github.com/chrislusf/seaweedfs/weed/glog" + "github.com/spf13/viper" ) const ( @@ -20,12 +20,11 @@ enabled = false enabled = false dir = "." # directory to store level db files -[mysql] +#################################################### # multiple filers on shared storage, fairly scalable -# -# need to choose or create a database. -# need to manually create a table "filemeta". -# +#################################################### + +[mysql] # CREATE TABLE IF NOT EXISTS filemeta ( # dirhash BIGINT COMMENT 'first 64 bits of MD5 hash value of directory field', # name VARCHAR(1000) COMMENT 'directory or file name', @@ -33,7 +32,6 @@ dir = "." # directory to store level db files # meta BLOB, # PRIMARY KEY (dirhash, name) # ) DEFAULT CHARSET=utf8; -# enabled = true hostname = "localhost" port = 3306 @@ -90,10 +88,10 @@ var ( func (f *Filer) LoadConfiguration() { // find a filer store - viper.SetConfigName("filer") // name of config file (without extension) - viper.AddConfigPath(".") // optionally look for config in the working directory - viper.AddConfigPath("$HOME/.seaweedfs") // call multiple times to add many search paths - viper.AddConfigPath("/etc/seaweedfs/") // path to look for the config file in + viper.SetConfigName("filer") // name of config file (without extension) + viper.AddConfigPath(".") // optionally look for config in the working directory + viper.AddConfigPath("$HOME/.seaweedfs") // call multiple times to add many search paths + viper.AddConfigPath("/etc/seaweedfs/") // path to look for the config file in if err := viper.ReadInConfig(); err != nil { // Handle errors reading the config file glog.Fatalf("Failed to load filer.toml file from current directory, or $HOME/.seaweedfs/, or /etc/seaweedfs/" + "\n\nPlease follow this example and add a filer.toml file to " + diff --git a/weed/filer2/entry.go b/weed/filer2/entry.go index f741d4bb3..cf7ad08aa 100644 --- a/weed/filer2/entry.go +++ b/weed/filer2/entry.go @@ -15,7 +15,7 @@ type Attr struct { Gid uint32 // group gid } -func (attr Attr) IsDirectory() (bool) { +func (attr Attr) IsDirectory() bool { return attr.Mode&os.ModeDir > 0 } diff --git a/weed/filer2/entry_codec.go b/weed/filer2/entry_codec.go index 7fafea3e3..8019b2193 100644 --- a/weed/filer2/entry_codec.go +++ b/weed/filer2/entry_codec.go @@ -4,9 +4,9 @@ import ( "os" "time" + "fmt" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "github.com/gogo/protobuf/proto" - "fmt" ) func (entry *Entry) EncodeAttributesAndChunks() ([]byte, error) { @@ -23,7 +23,7 @@ func (entry *Entry) EncodeAttributesAndChunks() ([]byte, error) { return proto.Marshal(message) } -func (entry *Entry) DecodeAttributesAndChunks(blob []byte) (error) { +func (entry *Entry) DecodeAttributesAndChunks(blob []byte) error { message := &filer_pb.Entry{} diff --git a/weed/filer2/filechunks.go b/weed/filer2/filechunks.go index 5877f2f71..86b1a85b5 100644 --- a/weed/filer2/filechunks.go +++ b/weed/filer2/filechunks.go @@ -1,9 +1,9 @@ package filer2 import ( - "sort" "log" "math" + "sort" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" ) diff --git a/weed/filer2/filechunks_test.go b/weed/filer2/filechunks_test.go index 033e66b1e..134bb9c19 100644 --- a/weed/filer2/filechunks_test.go +++ b/weed/filer2/filechunks_test.go @@ -1,8 +1,8 @@ package filer2 import ( - "testing" "log" + "testing" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" ) @@ -163,9 +163,9 @@ func TestChunksReading(t *testing.T) { Offset: 0, Size: 250, Expected: []*ChunkView{ - {Offset: 0, Size: 100, FileId: "abc", LogicOffset:0}, - {Offset: 0, Size: 100, FileId: "asdf", LogicOffset:100}, - {Offset: 0, Size: 50, FileId: "fsad", LogicOffset:200}, + {Offset: 0, Size: 100, FileId: "abc", LogicOffset: 0}, + {Offset: 0, Size: 100, FileId: "asdf", LogicOffset: 100}, + {Offset: 0, Size: 50, FileId: "fsad", LogicOffset: 200}, }, }, // case 1: updates overwrite full chunks @@ -177,7 +177,7 @@ func TestChunksReading(t *testing.T) { Offset: 50, Size: 100, Expected: []*ChunkView{ - {Offset: 50, Size: 100, FileId: "asdf", LogicOffset:50}, + {Offset: 50, Size: 100, FileId: "asdf", LogicOffset: 50}, }, }, // case 2: updates overwrite part of previous chunks @@ -189,8 +189,8 @@ func TestChunksReading(t *testing.T) { Offset: 25, Size: 50, Expected: []*ChunkView{ - {Offset: 25, Size: 25, FileId: "asdf", LogicOffset:25}, - {Offset: 0, Size: 25, FileId: "abc", LogicOffset:50}, + {Offset: 25, Size: 25, FileId: "asdf", LogicOffset: 25}, + {Offset: 0, Size: 25, FileId: "abc", LogicOffset: 50}, }, }, // case 3: updates overwrite full chunks @@ -203,8 +203,8 @@ func TestChunksReading(t *testing.T) { Offset: 0, Size: 200, Expected: []*ChunkView{ - {Offset: 0, Size: 50, FileId: "asdf", LogicOffset:0}, - {Offset: 0, Size: 150, FileId: "xxxx", LogicOffset:50}, + {Offset: 0, Size: 50, FileId: "asdf", LogicOffset: 0}, + {Offset: 0, Size: 150, FileId: "xxxx", LogicOffset: 50}, }, }, // case 4: updates far away from prev chunks @@ -217,7 +217,7 @@ func TestChunksReading(t *testing.T) { Offset: 0, Size: 400, Expected: []*ChunkView{ - {Offset: 0, Size: 200, FileId: "asdf", LogicOffset:0}, + {Offset: 0, Size: 200, FileId: "asdf", LogicOffset: 0}, // {Offset: 0, Size: 150, FileId: "xxxx"}, // missing intervals should not happen }, }, @@ -232,8 +232,8 @@ func TestChunksReading(t *testing.T) { Offset: 0, Size: 220, Expected: []*ChunkView{ - {Offset: 0, Size: 200, FileId: "asdf", LogicOffset:0}, - {Offset: 0, Size: 20, FileId: "abc", LogicOffset:200}, + {Offset: 0, Size: 200, FileId: "asdf", LogicOffset: 0}, + {Offset: 0, Size: 20, FileId: "abc", LogicOffset: 200}, }, }, // case 6: same updates @@ -246,7 +246,7 @@ func TestChunksReading(t *testing.T) { Offset: 0, Size: 100, Expected: []*ChunkView{ - {Offset: 0, Size: 100, FileId: "abc", LogicOffset:0}, + {Offset: 0, Size: 100, FileId: "abc", LogicOffset: 0}, }, }, // case 7: edge cases @@ -259,8 +259,8 @@ func TestChunksReading(t *testing.T) { Offset: 0, Size: 200, Expected: []*ChunkView{ - {Offset: 0, Size: 100, FileId: "abc", LogicOffset:0}, - {Offset: 0, Size: 100, FileId: "asdf", LogicOffset:100}, + {Offset: 0, Size: 100, FileId: "abc", LogicOffset: 0}, + {Offset: 0, Size: 100, FileId: "asdf", LogicOffset: 100}, }, }, } diff --git a/weed/filer2/filer.go b/weed/filer2/filer.go index 35d69b32e..f182adb13 100644 --- a/weed/filer2/filer.go +++ b/weed/filer2/filer.go @@ -3,12 +3,12 @@ package filer2 import ( "fmt" + "github.com/chrislusf/seaweedfs/weed/glog" "github.com/karlseguin/ccache" - "strings" + "os" "path/filepath" + "strings" "time" - "os" - "github.com/chrislusf/seaweedfs/weed/glog" ) type Filer struct { @@ -24,15 +24,15 @@ func NewFiler(master string) *Filer { } } -func (f *Filer) SetStore(store FilerStore) () { +func (f *Filer) SetStore(store FilerStore) { f.store = store } -func (f *Filer) DisableDirectoryCache() () { +func (f *Filer) DisableDirectoryCache() { f.directoryCache = nil } -func (f *Filer) CreateEntry(entry *Entry) (error) { +func (f *Filer) CreateEntry(entry *Entry) error { dirParts := strings.Split(string(entry.FullPath), "/") @@ -94,11 +94,11 @@ func (f *Filer) CreateEntry(entry *Entry) (error) { } /* - if !hasWritePermission(lastDirectoryEntry, entry) { - glog.V(0).Infof("directory %s: %v, entry: uid=%d gid=%d", - lastDirectoryEntry.FullPath, lastDirectoryEntry.Attr, entry.Uid, entry.Gid) - return fmt.Errorf("no write permission in folder %v", lastDirectoryEntry.FullPath) - } + if !hasWritePermission(lastDirectoryEntry, entry) { + glog.V(0).Infof("directory %s: %v, entry: uid=%d gid=%d", + lastDirectoryEntry.FullPath, lastDirectoryEntry.Attr, entry.Uid, entry.Gid) + return fmt.Errorf("no write permission in folder %v", lastDirectoryEntry.FullPath) + } */ if err := f.store.InsertEntry(entry); err != nil { @@ -135,12 +135,12 @@ func (f *Filer) DeleteEntry(p FullPath) (fileEntry *Entry, err error) { func (f *Filer) ListDirectoryEntries(p FullPath, startFileName string, inclusive bool, limit int) ([]*Entry, error) { if strings.HasSuffix(string(p), "/") && len(p) > 1 { - p = p[0:len(p)-1] + p = p[0 : len(p)-1] } return f.store.ListDirectoryEntries(p, startFileName, inclusive, limit) } -func (f *Filer) cacheGetDirectory(dirpath string) (*Entry) { +func (f *Filer) cacheGetDirectory(dirpath string) *Entry { if f.directoryCache == nil { return nil } diff --git a/weed/filer2/filerstore.go b/weed/filer2/filerstore.go index b9f7d1198..a5cd3352e 100644 --- a/weed/filer2/filerstore.go +++ b/weed/filer2/filerstore.go @@ -7,8 +7,8 @@ import ( type FilerStore interface { GetName() string - Initialize(viper *viper.Viper) (error) - InsertEntry(*Entry) (error) + Initialize(viper *viper.Viper) error + InsertEntry(*Entry) error UpdateEntry(*Entry) (err error) FindEntry(FullPath) (entry *Entry, err error) DeleteEntry(FullPath) (fileEntry *Entry, err error) diff --git a/weed/filer2/fullpath.go b/weed/filer2/fullpath.go index 20e42e9b9..be6e34431 100644 --- a/weed/filer2/fullpath.go +++ b/weed/filer2/fullpath.go @@ -25,7 +25,7 @@ func (fp FullPath) DirAndName() (string, string) { return dir[:len(dir)-1], name } -func (fp FullPath) Name() (string) { +func (fp FullPath) Name() string { _, name := filepath.Split(string(fp)) return name } diff --git a/weed/filer2/leveldb/leveldb_store.go b/weed/filer2/leveldb/leveldb_store.go index 8b2df93ac..a3125e923 100644 --- a/weed/filer2/leveldb/leveldb_store.go +++ b/weed/filer2/leveldb/leveldb_store.go @@ -1,15 +1,15 @@ package leveldb import ( - "fmt" "bytes" + "fmt" - "github.com/syndtr/goleveldb/leveldb" "github.com/chrislusf/seaweedfs/weed/filer2" - leveldb_util "github.com/syndtr/goleveldb/leveldb/util" "github.com/chrislusf/seaweedfs/weed/glog" - "github.com/spf13/viper" weed_util "github.com/chrislusf/seaweedfs/weed/util" + "github.com/spf13/viper" + "github.com/syndtr/goleveldb/leveldb" + leveldb_util "github.com/syndtr/goleveldb/leveldb/util" ) const ( @@ -162,7 +162,7 @@ func genDirectoryKeyPrefix(fullpath filer2.FullPath, startFileName string) (keyP return keyPrefix } -func getNameFromKey(key []byte) (string) { +func getNameFromKey(key []byte) string { sepIndex := len(key) - 1 for sepIndex >= 0 && key[sepIndex] != DIR_FILE_SEPARATOR { diff --git a/weed/filer2/leveldb/leveldb_store_test.go b/weed/filer2/leveldb/leveldb_store_test.go index 87f77d138..896dabdc3 100644 --- a/weed/filer2/leveldb/leveldb_store_test.go +++ b/weed/filer2/leveldb/leveldb_store_test.go @@ -1,10 +1,10 @@ package leveldb import ( - "testing" "github.com/chrislusf/seaweedfs/weed/filer2" "io/ioutil" "os" + "testing" ) func TestCreateAndFind(t *testing.T) { diff --git a/weed/filer2/memdb/memdb_store.go b/weed/filer2/memdb/memdb_store.go index 452e4970c..098ed5dce 100644 --- a/weed/filer2/memdb/memdb_store.go +++ b/weed/filer2/memdb/memdb_store.go @@ -1,11 +1,11 @@ package memdb import ( + "fmt" "github.com/chrislusf/seaweedfs/weed/filer2" "github.com/google/btree" - "strings" - "fmt" "github.com/spf13/viper" + "strings" ) func init() { diff --git a/weed/filer2/memdb/memdb_store_test.go b/weed/filer2/memdb/memdb_store_test.go index a8631f969..bd1cdfdc0 100644 --- a/weed/filer2/memdb/memdb_store_test.go +++ b/weed/filer2/memdb/memdb_store_test.go @@ -1,8 +1,8 @@ package memdb import ( - "testing" "github.com/chrislusf/seaweedfs/weed/filer2" + "testing" ) func TestCreateAndFind(t *testing.T) { diff --git a/weed/filer2/mysql/mysql_store.go b/weed/filer2/mysql/mysql_store.go index e56b961bb..475e4a642 100644 --- a/weed/filer2/mysql/mysql_store.go +++ b/weed/filer2/mysql/mysql_store.go @@ -1,13 +1,13 @@ package mysql import ( - "fmt" "database/sql" + "fmt" "github.com/chrislusf/seaweedfs/weed/filer2" - "github.com/spf13/viper" - _ "github.com/go-sql-driver/mysql" "github.com/chrislusf/seaweedfs/weed/filer2/abstract_sql" + _ "github.com/go-sql-driver/mysql" + "github.com/spf13/viper" ) const ( diff --git a/weed/filer2/postgres/postgres_store.go b/weed/filer2/postgres/postgres_store.go index 19d97b443..3bec55def 100644 --- a/weed/filer2/postgres/postgres_store.go +++ b/weed/filer2/postgres/postgres_store.go @@ -1,13 +1,13 @@ package postgres import ( - "fmt" "database/sql" + "fmt" "github.com/chrislusf/seaweedfs/weed/filer2" - "github.com/spf13/viper" - _ "github.com/lib/pq" "github.com/chrislusf/seaweedfs/weed/filer2/abstract_sql" + _ "github.com/lib/pq" + "github.com/spf13/viper" ) const ( diff --git a/weed/filer2/redis/redis_store.go b/weed/filer2/redis/redis_store.go index cb8381bf5..5d1f51812 100644 --- a/weed/filer2/redis/redis_store.go +++ b/weed/filer2/redis/redis_store.go @@ -4,8 +4,8 @@ import ( "fmt" "github.com/chrislusf/seaweedfs/weed/filer2" "github.com/chrislusf/seaweedfs/weed/glog" - "github.com/spf13/viper" "github.com/go-redis/redis" + "github.com/spf13/viper" "sort" "strings" ) diff --git a/weed/filesys/dir.go b/weed/filesys/dir.go index 6406dff95..e53c4cfaf 100644 --- a/weed/filesys/dir.go +++ b/weed/filesys/dir.go @@ -7,12 +7,12 @@ import ( "path" "sync" - "bazil.org/fuse/fs" "bazil.org/fuse" + "bazil.org/fuse/fs" "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" - "time" "path/filepath" + "time" ) type Dir struct { diff --git a/weed/filesys/file.go b/weed/filesys/file.go index d63f9d62d..7ea14cc49 100644 --- a/weed/filesys/file.go +++ b/weed/filesys/file.go @@ -1,15 +1,15 @@ package filesys import ( - "context" "bazil.org/fuse" "bazil.org/fuse/fs" + "context" + "github.com/chrislusf/seaweedfs/weed/filer2" "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" - "path/filepath" "os" + "path/filepath" "time" - "github.com/chrislusf/seaweedfs/weed/filer2" ) var _ = fs.Node(&File{}) diff --git a/weed/filesys/filehandle.go b/weed/filesys/filehandle.go index d9dc6795e..81aca42a4 100644 --- a/weed/filesys/filehandle.go +++ b/weed/filesys/filehandle.go @@ -1,19 +1,19 @@ package filesys import ( + "bazil.org/fuse" "bazil.org/fuse/fs" + "bytes" + "context" "fmt" - "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "github.com/chrislusf/seaweedfs/weed/filer2" - "context" "github.com/chrislusf/seaweedfs/weed/glog" - "bazil.org/fuse" - "bytes" "github.com/chrislusf/seaweedfs/weed/operation" - "time" + "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" + "github.com/chrislusf/seaweedfs/weed/util" "strings" "sync" - "github.com/chrislusf/seaweedfs/weed/util" + "time" ) type FileHandle struct { @@ -32,6 +32,7 @@ type FileHandle struct { } var _ = fs.Handle(&FileHandle{}) + // var _ = fs.HandleReadAller(&FileHandle{}) var _ = fs.HandleReader(&FileHandle{}) var _ = fs.HandleFlusher(&FileHandle{}) diff --git a/weed/filesys/wfs.go b/weed/filesys/wfs.go index 9f8f4649a..1b843e2d7 100644 --- a/weed/filesys/wfs.go +++ b/weed/filesys/wfs.go @@ -3,9 +3,9 @@ package filesys import ( "bazil.org/fuse/fs" "fmt" - "google.golang.org/grpc" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "github.com/karlseguin/ccache" + "google.golang.org/grpc" ) type WFS struct { @@ -15,7 +15,7 @@ type WFS struct { func NewSeaweedFileSystem(filer string) *WFS { return &WFS{ - filer: filer, + filer: filer, listDirectoryEntriesCache: ccache.New(ccache.Configure().MaxSize(6000).ItemsToPrune(100)), } } diff --git a/weed/images/favicon.go b/weed/images/favicon.go index 09504976c..2f0af200d 100644 --- a/weed/images/favicon.go +++ b/weed/images/favicon.go @@ -181,6 +181,7 @@ type bintree struct { Func func() (*asset, error) Children map[string]*bintree } + var _bintree = &bintree{nil, map[string]*bintree{ "favicon": &bintree{nil, map[string]*bintree{ "favicon.ico": &bintree{favicon, map[string]*bintree{}}, @@ -233,4 +234,3 @@ func _filePath(dir, name string) string { cannonicalName := strings.Replace(name, "\\", "/", -1) return filepath.Join(append([]string{dir}, strings.Split(cannonicalName, "/")...)...) } - diff --git a/weed/server/filer_grpc_server.go b/weed/server/filer_grpc_server.go index ffa24a0d8..44f5320cb 100644 --- a/weed/server/filer_grpc_server.go +++ b/weed/server/filer_grpc_server.go @@ -1,16 +1,16 @@ package weed_server import ( - "fmt" "context" - "time" + "fmt" "os" "path/filepath" + "time" - "github.com/chrislusf/seaweedfs/weed/operation" - "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "github.com/chrislusf/seaweedfs/weed/filer2" "github.com/chrislusf/seaweedfs/weed/glog" + "github.com/chrislusf/seaweedfs/weed/operation" + "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" ) func (fs *FilerServer) LookupDirectoryEntry(ctx context.Context, req *filer_pb.LookupDirectoryEntryRequest) (*filer_pb.LookupDirectoryEntryResponse, error) { diff --git a/weed/server/filer_server.go b/weed/server/filer_server.go index 012b0afbf..a1dc91650 100644 --- a/weed/server/filer_server.go +++ b/weed/server/filer_server.go @@ -7,10 +7,6 @@ import ( "sync" "time" - "github.com/chrislusf/seaweedfs/weed/glog" - "github.com/chrislusf/seaweedfs/weed/security" - "github.com/chrislusf/seaweedfs/weed/storage" - "github.com/chrislusf/seaweedfs/weed/util" "github.com/chrislusf/seaweedfs/weed/filer2" _ "github.com/chrislusf/seaweedfs/weed/filer2/cassandra" _ "github.com/chrislusf/seaweedfs/weed/filer2/leveldb" @@ -18,6 +14,10 @@ import ( _ "github.com/chrislusf/seaweedfs/weed/filer2/mysql" _ "github.com/chrislusf/seaweedfs/weed/filer2/postgres" _ "github.com/chrislusf/seaweedfs/weed/filer2/redis" + "github.com/chrislusf/seaweedfs/weed/glog" + "github.com/chrislusf/seaweedfs/weed/security" + "github.com/chrislusf/seaweedfs/weed/storage" + "github.com/chrislusf/seaweedfs/weed/util" ) type FilerServer struct { diff --git a/weed/server/filer_server_handlers_admin.go b/weed/server/filer_server_handlers_admin.go index f3cf2754f..91a0e6fa0 100644 --- a/weed/server/filer_server_handlers_admin.go +++ b/weed/server/filer_server_handlers_admin.go @@ -3,10 +3,10 @@ package weed_server import ( "net/http" - "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/filer2" - "strconv" + "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" + "strconv" "time" ) diff --git a/weed/server/filer_server_handlers_read.go b/weed/server/filer_server_handlers_read.go index e451a81de..782ed55f1 100644 --- a/weed/server/filer_server_handlers_read.go +++ b/weed/server/filer_server_handlers_read.go @@ -7,11 +7,11 @@ import ( "strconv" "strings" + "github.com/chrislusf/seaweedfs/weed/filer2" "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/operation" ui "github.com/chrislusf/seaweedfs/weed/server/filer_ui" "github.com/chrislusf/seaweedfs/weed/util" - "github.com/chrislusf/seaweedfs/weed/filer2" ) // listDirectoryHandler lists directories and folers under a directory diff --git a/weed/server/filer_server_handlers_write.go b/weed/server/filer_server_handlers_write.go index 9d5c620de..78028051e 100644 --- a/weed/server/filer_server_handlers_write.go +++ b/weed/server/filer_server_handlers_write.go @@ -17,13 +17,12 @@ import ( "strconv" "strings" - "github.com/chrislusf/seaweedfs/weed/filer" + "github.com/chrislusf/seaweedfs/weed/filer2" "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/operation" + "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "github.com/chrislusf/seaweedfs/weed/storage" "github.com/chrislusf/seaweedfs/weed/util" - "github.com/chrislusf/seaweedfs/weed/filer2" - "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "time" ) diff --git a/weed/server/filer_ui/breadcrumb.go b/weed/server/filer_ui/breadcrumb.go index 146b6c6c4..d056a4b25 100644 --- a/weed/server/filer_ui/breadcrumb.go +++ b/weed/server/filer_ui/breadcrumb.go @@ -1,8 +1,8 @@ package master_ui import ( - "strings" "path/filepath" + "strings" ) type Breadcrumb struct { diff --git a/weed/server/raft_server.go b/weed/server/raft_server.go index 591bc7caf..61adcdc59 100644 --- a/weed/server/raft_server.go +++ b/weed/server/raft_server.go @@ -198,7 +198,7 @@ func postFollowingOneRedirect(target string, contentType string, b bytes.Buffer) reply := string(data) if strings.HasPrefix(reply, "\"http") { - urlStr := reply[1: len(reply)-1] + urlStr := reply[1 : len(reply)-1] glog.V(0).Infoln("Post redirected to ", urlStr) resp2, err2 := http.Post(urlStr, contentType, backupReader) diff --git a/weed/server/volume_server_handlers_admin.go b/weed/server/volume_server_handlers_admin.go index 79bb89756..f66ab2306 100644 --- a/weed/server/volume_server_handlers_admin.go +++ b/weed/server/volume_server_handlers_admin.go @@ -8,8 +8,8 @@ import ( "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/stats" - "github.com/chrislusf/seaweedfs/weed/util" "github.com/chrislusf/seaweedfs/weed/storage" + "github.com/chrislusf/seaweedfs/weed/util" ) func (vs *VolumeServer) statusHandler(w http.ResponseWriter, r *http.Request) { diff --git a/weed/server/volume_server_handlers_ui.go b/weed/server/volume_server_handlers_ui.go index 7923c95c0..6fc775a6d 100644 --- a/weed/server/volume_server_handlers_ui.go +++ b/weed/server/volume_server_handlers_ui.go @@ -5,9 +5,9 @@ import ( "path/filepath" "time" + ui "github.com/chrislusf/seaweedfs/weed/server/volume_server_ui" "github.com/chrislusf/seaweedfs/weed/stats" "github.com/chrislusf/seaweedfs/weed/util" - ui "github.com/chrislusf/seaweedfs/weed/server/volume_server_ui" ) func (vs *VolumeServer) uiStatusHandler(w http.ResponseWriter, r *http.Request) { diff --git a/weed/storage/needle_map_memory.go b/weed/storage/needle_map_memory.go index 6da4bf82b..261486cf8 100644 --- a/weed/storage/needle_map_memory.go +++ b/weed/storage/needle_map_memory.go @@ -86,7 +86,7 @@ func WalkIndexFile(r *os.File, fn func(key uint64, offset, size uint32) error) e for count > 0 && e == nil || e == io.EOF { for i = 0; i+16 <= count; i += 16 { - key, offset, size = idxFileEntry(bytes[i: i+16]) + key, offset, size = idxFileEntry(bytes[i : i+16]) if e = fn(key, offset, size); e != nil { return e } diff --git a/weed/storage/volume_checking.go b/weed/storage/volume_checking.go index 67538ebb2..5603a878b 100644 --- a/weed/storage/volume_checking.go +++ b/weed/storage/volume_checking.go @@ -12,7 +12,7 @@ func getActualSize(size uint32) int64 { return NeedleHeaderSize + int64(size) + NeedleChecksumSize + int64(padding) } -func CheckVolumeDataIntegrity(v *Volume, indexFile *os.File) (error) { +func CheckVolumeDataIntegrity(v *Volume, indexFile *os.File) error { var indexSize int64 var e error if indexSize, e = verifyIndexFileIntegrity(indexFile); e != nil { From 8b0718ac92ff7f54db5e98ff175c7ac302a00df2 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sun, 27 May 2018 11:56:49 -0700 Subject: [PATCH 55/83] go vet --- weed/filer2/filechunks.go | 6 ++---- weed/topology/store_replicate.go | 1 - weed/util/http_util.go | 1 - 3 files changed, 2 insertions(+), 6 deletions(-) diff --git a/weed/filer2/filechunks.go b/weed/filer2/filechunks.go index 86b1a85b5..bd4a57f0f 100644 --- a/weed/filer2/filechunks.go +++ b/weed/filer2/filechunks.go @@ -1,7 +1,6 @@ package filer2 import ( - "log" "math" "sort" @@ -82,13 +81,12 @@ func ReadFromChunks(chunks []*filer_pb.FileChunk, offset int64, size int) (views } func logPrintf(name string, visibles []*visibleInterval) { - - return - + /* log.Printf("%s len %d", name, len(visibles)) for _, v := range visibles { log.Printf("%s: => %+v", name, v) } + */ } func nonOverlappingVisibleIntervals(chunks []*filer_pb.FileChunk) (visibles []*visibleInterval) { diff --git a/weed/topology/store_replicate.go b/weed/topology/store_replicate.go index d7fb501c0..1163c68d2 100644 --- a/weed/topology/store_replicate.go +++ b/weed/topology/store_replicate.go @@ -159,5 +159,4 @@ func distributedOperation(masterNode string, store *storage.Store, volumeId stor glog.V(0).Infoln() return fmt.Errorf("Failed to lookup for %d: %v", volumeId, lookupErr) } - return nil } diff --git a/weed/util/http_util.go b/weed/util/http_util.go index 6494041e3..579abaac0 100644 --- a/weed/util/http_util.go +++ b/weed/util/http_util.go @@ -141,7 +141,6 @@ func GetBufferStream(url string, values url.Values, allocatedBytes []byte, eachB return err } } - return nil } func GetUrlStream(url string, values url.Values, readFn func(io.Reader) error) error { From 5b844d70113c414724e7ada2d57c291087173a84 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sun, 27 May 2018 11:58:00 -0700 Subject: [PATCH 56/83] pass reference of master server SendHeartbeat passes lock by value: weed_server.MasterServer contains sync.Mutex --- weed/server/master_grpc_server.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/weed/server/master_grpc_server.go b/weed/server/master_grpc_server.go index 12ef9e927..d9b8f9e09 100644 --- a/weed/server/master_grpc_server.go +++ b/weed/server/master_grpc_server.go @@ -11,7 +11,7 @@ import ( "google.golang.org/grpc/peer" ) -func (ms MasterServer) SendHeartbeat(stream master_pb.Seaweed_SendHeartbeatServer) error { +func (ms *MasterServer) SendHeartbeat(stream master_pb.Seaweed_SendHeartbeatServer) error { var dn *topology.DataNode t := ms.Topo for { From 480a073f1fa01044328c1e14662367d9fc1539ad Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sun, 27 May 2018 23:53:10 -0700 Subject: [PATCH 57/83] refactoring: split into 4 files --- weed/server/filer_server_handlers_write.go | 331 ------------------ .../filer_server_handlers_write_autochunk.go | 186 ++++++++++ .../filer_server_handlers_write_monopart.go | 67 ++++ .../filer_server_handlers_write_multipart.go | 112 ++++++ 4 files changed, 365 insertions(+), 331 deletions(-) create mode 100644 weed/server/filer_server_handlers_write_autochunk.go create mode 100644 weed/server/filer_server_handlers_write_monopart.go create mode 100644 weed/server/filer_server_handlers_write_multipart.go diff --git a/weed/server/filer_server_handlers_write.go b/weed/server/filer_server_handlers_write.go index 78028051e..53b8d9e67 100644 --- a/weed/server/filer_server_handlers_write.go +++ b/weed/server/filer_server_handlers_write.go @@ -1,19 +1,11 @@ package weed_server import ( - "bytes" - "crypto/md5" - "encoding/base64" "encoding/json" "errors" - "fmt" - "io" "io/ioutil" - "mime/multipart" "net/http" - "net/textproto" "net/url" - "path" "strconv" "strings" @@ -21,7 +13,6 @@ import ( "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/operation" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" - "github.com/chrislusf/seaweedfs/weed/storage" "github.com/chrislusf/seaweedfs/weed/util" "time" ) @@ -34,46 +25,6 @@ type FilerPostResult struct { Url string `json:"url,omitempty"` } -var quoteEscaper = strings.NewReplacer("\\", "\\\\", `"`, "\\\"") - -func escapeQuotes(s string) string { - return quoteEscaper.Replace(s) -} - -func createFormFile(writer *multipart.Writer, fieldname, filename, mime string) (io.Writer, error) { - h := make(textproto.MIMEHeader) - h.Set("Content-Disposition", - fmt.Sprintf(`form-data; name="%s"; filename="%s"`, - escapeQuotes(fieldname), escapeQuotes(filename))) - if len(mime) == 0 { - mime = "application/octet-stream" - } - h.Set("Content-Type", mime) - return writer.CreatePart(h) -} - -func makeFormData(filename, mimeType string, content io.Reader) (formData io.Reader, contentType string, err error) { - buf := new(bytes.Buffer) - writer := multipart.NewWriter(buf) - defer writer.Close() - - part, err := createFormFile(writer, "file", filename, mimeType) - if err != nil { - glog.V(0).Infoln(err) - return - } - _, err = io.Copy(part, content) - if err != nil { - glog.V(0).Infoln(err) - return - } - - formData = buf - contentType = writer.FormDataContentType() - - return -} - func (fs *FilerServer) queryFileInfoByPath(w http.ResponseWriter, r *http.Request, path string) (fileId, urlLocation string, err error) { var entry *filer2.Entry if entry, err = fs.filer.FindEntry(filer2.FullPath(path)); err != nil { @@ -109,117 +60,6 @@ func (fs *FilerServer) assignNewFileInfo(w http.ResponseWriter, r *http.Request, return } -func (fs *FilerServer) multipartUploadAnalyzer(w http.ResponseWriter, r *http.Request, replication, collection string) (fileId, urlLocation string, err error) { - //Default handle way for http multipart - if r.Method == "PUT" { - buf, _ := ioutil.ReadAll(r.Body) - r.Body = ioutil.NopCloser(bytes.NewBuffer(buf)) - fileName, _, _, _, _, _, _, _, pe := storage.ParseUpload(r) - if pe != nil { - glog.V(0).Infoln("failing to parse post body", pe.Error()) - writeJsonError(w, r, http.StatusInternalServerError, pe) - err = pe - return - } - //reconstruct http request body for following new request to volume server - r.Body = ioutil.NopCloser(bytes.NewBuffer(buf)) - - path := r.URL.Path - if strings.HasSuffix(path, "/") { - if fileName != "" { - path += fileName - } - } - fileId, urlLocation, err = fs.queryFileInfoByPath(w, r, path) - } else { - fileId, urlLocation, err = fs.assignNewFileInfo(w, r, replication, collection) - } - return -} - -func multipartHttpBodyBuilder(w http.ResponseWriter, r *http.Request, fileName string) (err error) { - body, contentType, te := makeFormData(fileName, r.Header.Get("Content-Type"), r.Body) - if te != nil { - glog.V(0).Infoln("S3 protocol to raw seaweed protocol failed", te.Error()) - writeJsonError(w, r, http.StatusInternalServerError, te) - err = te - return - } - - if body != nil { - switch v := body.(type) { - case *bytes.Buffer: - r.ContentLength = int64(v.Len()) - case *bytes.Reader: - r.ContentLength = int64(v.Len()) - case *strings.Reader: - r.ContentLength = int64(v.Len()) - } - } - - r.Header.Set("Content-Type", contentType) - rc, ok := body.(io.ReadCloser) - if !ok && body != nil { - rc = ioutil.NopCloser(body) - } - r.Body = rc - return -} - -func checkContentMD5(w http.ResponseWriter, r *http.Request) (err error) { - if contentMD5 := r.Header.Get("Content-MD5"); contentMD5 != "" { - buf, _ := ioutil.ReadAll(r.Body) - //checkMD5 - sum := md5.Sum(buf) - fileDataMD5 := base64.StdEncoding.EncodeToString(sum[0:len(sum)]) - if strings.ToLower(fileDataMD5) != strings.ToLower(contentMD5) { - glog.V(0).Infof("fileDataMD5 [%s] is not equal to Content-MD5 [%s]", fileDataMD5, contentMD5) - err = fmt.Errorf("MD5 check failed") - writeJsonError(w, r, http.StatusNotAcceptable, err) - return - } - //reconstruct http request body for following new request to volume server - r.Body = ioutil.NopCloser(bytes.NewBuffer(buf)) - } - return -} - -func (fs *FilerServer) monolithicUploadAnalyzer(w http.ResponseWriter, r *http.Request, replication, collection string) (fileId, urlLocation string, err error) { - /* - Amazon S3 ref link:[http://docs.aws.amazon.com/AmazonS3/latest/API/Welcome.html] - There is a long way to provide a completely compatibility against all Amazon S3 API, I just made - a simple data stream adapter between S3 PUT API and seaweedfs's volume storage Write API - 1. The request url format should be http://$host:$port/$bucketName/$objectName - 2. bucketName will be mapped to seaweedfs's collection name - 3. You could customize and make your enhancement. - */ - lastPos := strings.LastIndex(r.URL.Path, "/") - if lastPos == -1 || lastPos == 0 || lastPos == len(r.URL.Path)-1 { - glog.V(0).Infoln("URL Path [%s] is invalid, could not retrieve file name", r.URL.Path) - err = fmt.Errorf("URL Path is invalid") - writeJsonError(w, r, http.StatusInternalServerError, err) - return - } - - if err = checkContentMD5(w, r); err != nil { - return - } - - fileName := r.URL.Path[lastPos+1:] - if err = multipartHttpBodyBuilder(w, r, fileName); err != nil { - return - } - - secondPos := strings.Index(r.URL.Path[1:], "/") + 1 - collection = r.URL.Path[1:secondPos] - path := r.URL.Path - - if fileId, urlLocation, err = fs.queryFileInfoByPath(w, r, path); err == nil && fileId == "" { - fileId, urlLocation, err = fs.assignNewFileInfo(w, r, replication, collection) - } - return -} - func (fs *FilerServer) PostHandler(w http.ResponseWriter, r *http.Request) { query := r.URL.Query() @@ -352,177 +192,6 @@ func (fs *FilerServer) PostHandler(w http.ResponseWriter, r *http.Request) { writeJsonQuiet(w, r, http.StatusCreated, reply) } -func (fs *FilerServer) autoChunk(w http.ResponseWriter, r *http.Request, replication string, collection string) bool { - if r.Method != "POST" { - glog.V(4).Infoln("AutoChunking not supported for method", r.Method) - return false - } - - // autoChunking can be set at the command-line level or as a query param. Query param overrides command-line - query := r.URL.Query() - - parsedMaxMB, _ := strconv.ParseInt(query.Get("maxMB"), 10, 32) - maxMB := int32(parsedMaxMB) - if maxMB <= 0 && fs.maxMB > 0 { - maxMB = int32(fs.maxMB) - } - if maxMB <= 0 { - glog.V(4).Infoln("AutoChunking not enabled") - return false - } - glog.V(4).Infoln("AutoChunking level set to", maxMB, "(MB)") - - chunkSize := 1024 * 1024 * maxMB - - contentLength := int64(0) - if contentLengthHeader := r.Header["Content-Length"]; len(contentLengthHeader) == 1 { - contentLength, _ = strconv.ParseInt(contentLengthHeader[0], 10, 64) - if contentLength <= int64(chunkSize) { - glog.V(4).Infoln("Content-Length of", contentLength, "is less than the chunk size of", chunkSize, "so autoChunking will be skipped.") - return false - } - } - - if contentLength <= 0 { - glog.V(4).Infoln("Content-Length value is missing or unexpected so autoChunking will be skipped.") - return false - } - - reply, err := fs.doAutoChunk(w, r, contentLength, chunkSize, replication, collection) - if err != nil { - writeJsonError(w, r, http.StatusInternalServerError, err) - } else if reply != nil { - writeJsonQuiet(w, r, http.StatusCreated, reply) - } - return true -} - -func (fs *FilerServer) doAutoChunk(w http.ResponseWriter, r *http.Request, contentLength int64, chunkSize int32, replication string, collection string) (filerResult *FilerPostResult, replyerr error) { - - multipartReader, multipartReaderErr := r.MultipartReader() - if multipartReaderErr != nil { - return nil, multipartReaderErr - } - - part1, part1Err := multipartReader.NextPart() - if part1Err != nil { - return nil, part1Err - } - - fileName := part1.FileName() - if fileName != "" { - fileName = path.Base(fileName) - } - - var fileChunks []*filer_pb.FileChunk - - totalBytesRead := int64(0) - tmpBufferSize := int32(1024 * 1024) - tmpBuffer := bytes.NewBuffer(make([]byte, 0, tmpBufferSize)) - chunkBuf := make([]byte, chunkSize+tmpBufferSize, chunkSize+tmpBufferSize) // chunk size plus a little overflow - chunkBufOffset := int32(0) - chunkOffset := int64(0) - writtenChunks := 0 - - filerResult = &FilerPostResult{ - Name: fileName, - } - - for totalBytesRead < contentLength { - tmpBuffer.Reset() - bytesRead, readErr := io.CopyN(tmpBuffer, part1, int64(tmpBufferSize)) - readFully := readErr != nil && readErr == io.EOF - tmpBuf := tmpBuffer.Bytes() - bytesToCopy := tmpBuf[0:int(bytesRead)] - - copy(chunkBuf[chunkBufOffset:chunkBufOffset+int32(bytesRead)], bytesToCopy) - chunkBufOffset = chunkBufOffset + int32(bytesRead) - - if chunkBufOffset >= chunkSize || readFully || (chunkBufOffset > 0 && bytesRead == 0) { - writtenChunks = writtenChunks + 1 - fileId, urlLocation, assignErr := fs.assignNewFileInfo(w, r, replication, collection) - if assignErr != nil { - return nil, assignErr - } - - // upload the chunk to the volume server - chunkName := fileName + "_chunk_" + strconv.FormatInt(int64(len(fileChunks)+1), 10) - uploadErr := fs.doUpload(urlLocation, w, r, chunkBuf[0:chunkBufOffset], chunkName, "application/octet-stream", fileId) - if uploadErr != nil { - return nil, uploadErr - } - - // Save to chunk manifest structure - fileChunks = append(fileChunks, - &filer_pb.FileChunk{ - FileId: fileId, - Offset: chunkOffset, - Size: uint64(chunkBufOffset), - Mtime: time.Now().UnixNano(), - }, - ) - - // reset variables for the next chunk - chunkBufOffset = 0 - chunkOffset = totalBytesRead + int64(bytesRead) - } - - totalBytesRead = totalBytesRead + int64(bytesRead) - - if bytesRead == 0 || readFully { - break - } - - if readErr != nil { - return nil, readErr - } - } - - path := r.URL.Path - // also delete the old fid unless PUT operation - if r.Method != "PUT" { - if entry, err := fs.filer.FindEntry(filer2.FullPath(path)); err == nil { - for _, chunk := range entry.Chunks { - oldFid := chunk.FileId - operation.DeleteFile(fs.getMasterNode(), oldFid, fs.jwt(oldFid)) - } - } else if err != nil { - glog.V(0).Infof("error %v occur when finding %s in filer store", err, path) - } - } - - glog.V(4).Infoln("saving", path) - entry := &filer2.Entry{ - FullPath: filer2.FullPath(path), - Attr: filer2.Attr{ - Mode: 0660, - }, - Chunks: fileChunks, - } - if db_err := fs.filer.CreateEntry(entry); db_err != nil { - replyerr = db_err - filerResult.Error = db_err.Error() - glog.V(0).Infof("failing to write %s to filer server : %v", path, db_err) - return - } - - return -} - -func (fs *FilerServer) doUpload(urlLocation string, w http.ResponseWriter, r *http.Request, chunkBuf []byte, fileName string, contentType string, fileId string) (err error) { - err = nil - - ioReader := ioutil.NopCloser(bytes.NewBuffer(chunkBuf)) - uploadResult, uploadError := operation.Upload(urlLocation, fileName, ioReader, false, contentType, nil, fs.jwt(fileId)) - if uploadResult != nil { - glog.V(0).Infoln("Chunk upload result. Name:", uploadResult.Name, "Fid:", fileId, "Size:", uploadResult.Size) - } - if uploadError != nil { - err = uploadError - } - return -} - // curl -X DELETE http://localhost:8888/path/to func (fs *FilerServer) DeleteHandler(w http.ResponseWriter, r *http.Request) { diff --git a/weed/server/filer_server_handlers_write_autochunk.go b/weed/server/filer_server_handlers_write_autochunk.go new file mode 100644 index 000000000..76a720e49 --- /dev/null +++ b/weed/server/filer_server_handlers_write_autochunk.go @@ -0,0 +1,186 @@ +package weed_server + +import ( + "bytes" + "io" + "net/http" + "path" + "strconv" + "github.com/chrislusf/seaweedfs/weed/filer2" + "github.com/chrislusf/seaweedfs/weed/glog" + "github.com/chrislusf/seaweedfs/weed/operation" + "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" + "time" + "io/ioutil" +) + +func (fs *FilerServer) autoChunk(w http.ResponseWriter, r *http.Request, replication string, collection string) bool { + if r.Method != "POST" { + glog.V(4).Infoln("AutoChunking not supported for method", r.Method) + return false + } + + // autoChunking can be set at the command-line level or as a query param. Query param overrides command-line + query := r.URL.Query() + + parsedMaxMB, _ := strconv.ParseInt(query.Get("maxMB"), 10, 32) + maxMB := int32(parsedMaxMB) + if maxMB <= 0 && fs.maxMB > 0 { + maxMB = int32(fs.maxMB) + } + if maxMB <= 0 { + glog.V(4).Infoln("AutoChunking not enabled") + return false + } + glog.V(4).Infoln("AutoChunking level set to", maxMB, "(MB)") + + chunkSize := 1024 * 1024 * maxMB + + contentLength := int64(0) + if contentLengthHeader := r.Header["Content-Length"]; len(contentLengthHeader) == 1 { + contentLength, _ = strconv.ParseInt(contentLengthHeader[0], 10, 64) + if contentLength <= int64(chunkSize) { + glog.V(4).Infoln("Content-Length of", contentLength, "is less than the chunk size of", chunkSize, "so autoChunking will be skipped.") + return false + } + } + + if contentLength <= 0 { + glog.V(4).Infoln("Content-Length value is missing or unexpected so autoChunking will be skipped.") + return false + } + + reply, err := fs.doAutoChunk(w, r, contentLength, chunkSize, replication, collection) + if err != nil { + writeJsonError(w, r, http.StatusInternalServerError, err) + } else if reply != nil { + writeJsonQuiet(w, r, http.StatusCreated, reply) + } + return true +} + +func (fs *FilerServer) doAutoChunk(w http.ResponseWriter, r *http.Request, contentLength int64, chunkSize int32, replication string, collection string) (filerResult *FilerPostResult, replyerr error) { + + multipartReader, multipartReaderErr := r.MultipartReader() + if multipartReaderErr != nil { + return nil, multipartReaderErr + } + + part1, part1Err := multipartReader.NextPart() + if part1Err != nil { + return nil, part1Err + } + + fileName := part1.FileName() + if fileName != "" { + fileName = path.Base(fileName) + } + + var fileChunks []*filer_pb.FileChunk + + totalBytesRead := int64(0) + tmpBufferSize := int32(1024 * 1024) + tmpBuffer := bytes.NewBuffer(make([]byte, 0, tmpBufferSize)) + chunkBuf := make([]byte, chunkSize+tmpBufferSize, chunkSize+tmpBufferSize) // chunk size plus a little overflow + chunkBufOffset := int32(0) + chunkOffset := int64(0) + writtenChunks := 0 + + filerResult = &FilerPostResult{ + Name: fileName, + } + + for totalBytesRead < contentLength { + tmpBuffer.Reset() + bytesRead, readErr := io.CopyN(tmpBuffer, part1, int64(tmpBufferSize)) + readFully := readErr != nil && readErr == io.EOF + tmpBuf := tmpBuffer.Bytes() + bytesToCopy := tmpBuf[0:int(bytesRead)] + + copy(chunkBuf[chunkBufOffset:chunkBufOffset+int32(bytesRead)], bytesToCopy) + chunkBufOffset = chunkBufOffset + int32(bytesRead) + + if chunkBufOffset >= chunkSize || readFully || (chunkBufOffset > 0 && bytesRead == 0) { + writtenChunks = writtenChunks + 1 + fileId, urlLocation, assignErr := fs.assignNewFileInfo(w, r, replication, collection) + if assignErr != nil { + return nil, assignErr + } + + // upload the chunk to the volume server + chunkName := fileName + "_chunk_" + strconv.FormatInt(int64(len(fileChunks)+1), 10) + uploadErr := fs.doUpload(urlLocation, w, r, chunkBuf[0:chunkBufOffset], chunkName, "application/octet-stream", fileId) + if uploadErr != nil { + return nil, uploadErr + } + + // Save to chunk manifest structure + fileChunks = append(fileChunks, + &filer_pb.FileChunk{ + FileId: fileId, + Offset: chunkOffset, + Size: uint64(chunkBufOffset), + Mtime: time.Now().UnixNano(), + }, + ) + + // reset variables for the next chunk + chunkBufOffset = 0 + chunkOffset = totalBytesRead + int64(bytesRead) + } + + totalBytesRead = totalBytesRead + int64(bytesRead) + + if bytesRead == 0 || readFully { + break + } + + if readErr != nil { + return nil, readErr + } + } + + path := r.URL.Path + // also delete the old fid unless PUT operation + if r.Method != "PUT" { + if entry, err := fs.filer.FindEntry(filer2.FullPath(path)); err == nil { + for _, chunk := range entry.Chunks { + oldFid := chunk.FileId + operation.DeleteFile(fs.getMasterNode(), oldFid, fs.jwt(oldFid)) + } + } else if err != nil { + glog.V(0).Infof("error %v occur when finding %s in filer store", err, path) + } + } + + glog.V(4).Infoln("saving", path) + entry := &filer2.Entry{ + FullPath: filer2.FullPath(path), + Attr: filer2.Attr{ + Mode: 0660, + }, + Chunks: fileChunks, + } + if db_err := fs.filer.CreateEntry(entry); db_err != nil { + replyerr = db_err + filerResult.Error = db_err.Error() + glog.V(0).Infof("failing to write %s to filer server : %v", path, db_err) + return + } + + return +} + +func (fs *FilerServer) doUpload(urlLocation string, w http.ResponseWriter, r *http.Request, chunkBuf []byte, fileName string, contentType string, fileId string) (err error) { + err = nil + + ioReader := ioutil.NopCloser(bytes.NewBuffer(chunkBuf)) + uploadResult, uploadError := operation.Upload(urlLocation, fileName, ioReader, false, contentType, nil, fs.jwt(fileId)) + if uploadResult != nil { + glog.V(0).Infoln("Chunk upload result. Name:", uploadResult.Name, "Fid:", fileId, "Size:", uploadResult.Size) + } + if uploadError != nil { + err = uploadError + } + return +} diff --git a/weed/server/filer_server_handlers_write_monopart.go b/weed/server/filer_server_handlers_write_monopart.go new file mode 100644 index 000000000..550d32aed --- /dev/null +++ b/weed/server/filer_server_handlers_write_monopart.go @@ -0,0 +1,67 @@ +package weed_server + +import ( + "bytes" + "crypto/md5" + "encoding/base64" + "fmt" + "io/ioutil" + "net/http" + "strings" + + "github.com/chrislusf/seaweedfs/weed/glog" +) + +func checkContentMD5(w http.ResponseWriter, r *http.Request) (err error) { + if contentMD5 := r.Header.Get("Content-MD5"); contentMD5 != "" { + buf, _ := ioutil.ReadAll(r.Body) + //checkMD5 + sum := md5.Sum(buf) + fileDataMD5 := base64.StdEncoding.EncodeToString(sum[0:len(sum)]) + if strings.ToLower(fileDataMD5) != strings.ToLower(contentMD5) { + glog.V(0).Infof("fileDataMD5 [%s] is not equal to Content-MD5 [%s]", fileDataMD5, contentMD5) + err = fmt.Errorf("MD5 check failed") + writeJsonError(w, r, http.StatusNotAcceptable, err) + return + } + //reconstruct http request body for following new request to volume server + r.Body = ioutil.NopCloser(bytes.NewBuffer(buf)) + } + return +} + +func (fs *FilerServer) monolithicUploadAnalyzer(w http.ResponseWriter, r *http.Request, replication, collection string) (fileId, urlLocation string, err error) { + /* + Amazon S3 ref link:[http://docs.aws.amazon.com/AmazonS3/latest/API/Welcome.html] + There is a long way to provide a completely compatibility against all Amazon S3 API, I just made + a simple data stream adapter between S3 PUT API and seaweedfs's volume storage Write API + 1. The request url format should be http://$host:$port/$bucketName/$objectName + 2. bucketName will be mapped to seaweedfs's collection name + 3. You could customize and make your enhancement. + */ + lastPos := strings.LastIndex(r.URL.Path, "/") + if lastPos == -1 || lastPos == 0 || lastPos == len(r.URL.Path)-1 { + glog.V(0).Infoln("URL Path [%s] is invalid, could not retrieve file name", r.URL.Path) + err = fmt.Errorf("URL Path is invalid") + writeJsonError(w, r, http.StatusInternalServerError, err) + return + } + + if err = checkContentMD5(w, r); err != nil { + return + } + + fileName := r.URL.Path[lastPos+1:] + if err = multipartHttpBodyBuilder(w, r, fileName); err != nil { + return + } + + secondPos := strings.Index(r.URL.Path[1:], "/") + 1 + collection = r.URL.Path[1:secondPos] + path := r.URL.Path + + if fileId, urlLocation, err = fs.queryFileInfoByPath(w, r, path); err == nil && fileId == "" { + fileId, urlLocation, err = fs.assignNewFileInfo(w, r, replication, collection) + } + return +} diff --git a/weed/server/filer_server_handlers_write_multipart.go b/weed/server/filer_server_handlers_write_multipart.go new file mode 100644 index 000000000..edf9305c6 --- /dev/null +++ b/weed/server/filer_server_handlers_write_multipart.go @@ -0,0 +1,112 @@ +package weed_server + +import ( + "bytes" + "fmt" + "io" + "io/ioutil" + "mime/multipart" + "net/http" + "net/textproto" + "strings" + + "github.com/chrislusf/seaweedfs/weed/glog" + "github.com/chrislusf/seaweedfs/weed/storage" +) + +var quoteEscaper = strings.NewReplacer("\\", "\\\\", `"`, "\\\"") + +func escapeQuotes(s string) string { + return quoteEscaper.Replace(s) +} + +func createFormFile(writer *multipart.Writer, fieldname, filename, mime string) (io.Writer, error) { + h := make(textproto.MIMEHeader) + h.Set("Content-Disposition", + fmt.Sprintf(`form-data; name="%s"; filename="%s"`, + escapeQuotes(fieldname), escapeQuotes(filename))) + if len(mime) == 0 { + mime = "application/octet-stream" + } + h.Set("Content-Type", mime) + return writer.CreatePart(h) +} + +func makeFormData(filename, mimeType string, content io.Reader) (formData io.Reader, contentType string, err error) { + buf := new(bytes.Buffer) + writer := multipart.NewWriter(buf) + defer writer.Close() + + part, err := createFormFile(writer, "file", filename, mimeType) + if err != nil { + glog.V(0).Infoln(err) + return + } + _, err = io.Copy(part, content) + if err != nil { + glog.V(0).Infoln(err) + return + } + + formData = buf + contentType = writer.FormDataContentType() + + return +} + +func (fs *FilerServer) multipartUploadAnalyzer(w http.ResponseWriter, r *http.Request, replication, collection string) (fileId, urlLocation string, err error) { + //Default handle way for http multipart + if r.Method == "PUT" { + buf, _ := ioutil.ReadAll(r.Body) + r.Body = ioutil.NopCloser(bytes.NewBuffer(buf)) + fileName, _, _, _, _, _, _, _, pe := storage.ParseUpload(r) + if pe != nil { + glog.V(0).Infoln("failing to parse post body", pe.Error()) + writeJsonError(w, r, http.StatusInternalServerError, pe) + err = pe + return + } + //reconstruct http request body for following new request to volume server + r.Body = ioutil.NopCloser(bytes.NewBuffer(buf)) + + path := r.URL.Path + if strings.HasSuffix(path, "/") { + if fileName != "" { + path += fileName + } + } + fileId, urlLocation, err = fs.queryFileInfoByPath(w, r, path) + } else { + fileId, urlLocation, err = fs.assignNewFileInfo(w, r, replication, collection) + } + return +} + +func multipartHttpBodyBuilder(w http.ResponseWriter, r *http.Request, fileName string) (err error) { + body, contentType, te := makeFormData(fileName, r.Header.Get("Content-Type"), r.Body) + if te != nil { + glog.V(0).Infoln("S3 protocol to raw seaweed protocol failed", te.Error()) + writeJsonError(w, r, http.StatusInternalServerError, te) + err = te + return + } + + if body != nil { + switch v := body.(type) { + case *bytes.Buffer: + r.ContentLength = int64(v.Len()) + case *bytes.Reader: + r.ContentLength = int64(v.Len()) + case *strings.Reader: + r.ContentLength = int64(v.Len()) + } + } + + r.Header.Set("Content-Type", contentType) + rc, ok := body.(io.ReadCloser) + if !ok && body != nil { + rc = ioutil.NopCloser(body) + } + r.Body = rc + return +} From 8db9319a064c8151677c2ccdbc06b60d476f59b4 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Sun, 27 May 2018 23:59:49 -0700 Subject: [PATCH 58/83] refactoring: go fmt, reorg --- weed/filer2/filechunks.go | 8 +- weed/server/filer_server_handlers_write.go | 2 +- .../filer_server_handlers_write_autochunk.go | 5 +- .../filer_server_handlers_write_monopart.go | 72 ++++++++++++++++++ .../filer_server_handlers_write_multipart.go | 73 ------------------- 5 files changed, 80 insertions(+), 80 deletions(-) diff --git a/weed/filer2/filechunks.go b/weed/filer2/filechunks.go index bd4a57f0f..c4c77d270 100644 --- a/weed/filer2/filechunks.go +++ b/weed/filer2/filechunks.go @@ -82,10 +82,10 @@ func ReadFromChunks(chunks []*filer_pb.FileChunk, offset int64, size int) (views func logPrintf(name string, visibles []*visibleInterval) { /* - log.Printf("%s len %d", name, len(visibles)) - for _, v := range visibles { - log.Printf("%s: => %+v", name, v) - } + log.Printf("%s len %d", name, len(visibles)) + for _, v := range visibles { + log.Printf("%s: => %+v", name, v) + } */ } diff --git a/weed/server/filer_server_handlers_write.go b/weed/server/filer_server_handlers_write.go index 53b8d9e67..c4152ba4f 100644 --- a/weed/server/filer_server_handlers_write.go +++ b/weed/server/filer_server_handlers_write.go @@ -8,13 +8,13 @@ import ( "net/url" "strconv" "strings" + "time" "github.com/chrislusf/seaweedfs/weed/filer2" "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/operation" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "github.com/chrislusf/seaweedfs/weed/util" - "time" ) type FilerPostResult struct { diff --git a/weed/server/filer_server_handlers_write_autochunk.go b/weed/server/filer_server_handlers_write_autochunk.go index 76a720e49..a1f0fa27f 100644 --- a/weed/server/filer_server_handlers_write_autochunk.go +++ b/weed/server/filer_server_handlers_write_autochunk.go @@ -3,15 +3,16 @@ package weed_server import ( "bytes" "io" + "io/ioutil" "net/http" "path" "strconv" + "time" + "github.com/chrislusf/seaweedfs/weed/filer2" "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/operation" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" - "time" - "io/ioutil" ) func (fs *FilerServer) autoChunk(w http.ResponseWriter, r *http.Request, replication string, collection string) bool { diff --git a/weed/server/filer_server_handlers_write_monopart.go b/weed/server/filer_server_handlers_write_monopart.go index 550d32aed..30fbcf7f9 100644 --- a/weed/server/filer_server_handlers_write_monopart.go +++ b/weed/server/filer_server_handlers_write_monopart.go @@ -5,13 +5,56 @@ import ( "crypto/md5" "encoding/base64" "fmt" + "io" "io/ioutil" + "mime/multipart" "net/http" + "net/textproto" "strings" "github.com/chrislusf/seaweedfs/weed/glog" ) +var quoteEscaper = strings.NewReplacer("\\", "\\\\", `"`, "\\\"") + +func escapeQuotes(s string) string { + return quoteEscaper.Replace(s) +} + +func createFormFile(writer *multipart.Writer, fieldname, filename, mime string) (io.Writer, error) { + h := make(textproto.MIMEHeader) + h.Set("Content-Disposition", + fmt.Sprintf(`form-data; name="%s"; filename="%s"`, + escapeQuotes(fieldname), escapeQuotes(filename))) + if len(mime) == 0 { + mime = "application/octet-stream" + } + h.Set("Content-Type", mime) + return writer.CreatePart(h) +} + +func makeFormData(filename, mimeType string, content io.Reader) (formData io.Reader, contentType string, err error) { + buf := new(bytes.Buffer) + writer := multipart.NewWriter(buf) + defer writer.Close() + + part, err := createFormFile(writer, "file", filename, mimeType) + if err != nil { + glog.V(0).Infoln(err) + return + } + _, err = io.Copy(part, content) + if err != nil { + glog.V(0).Infoln(err) + return + } + + formData = buf + contentType = writer.FormDataContentType() + + return +} + func checkContentMD5(w http.ResponseWriter, r *http.Request) (err error) { if contentMD5 := r.Header.Get("Content-MD5"); contentMD5 != "" { buf, _ := ioutil.ReadAll(r.Body) @@ -65,3 +108,32 @@ func (fs *FilerServer) monolithicUploadAnalyzer(w http.ResponseWriter, r *http.R } return } + +func multipartHttpBodyBuilder(w http.ResponseWriter, r *http.Request, fileName string) (err error) { + body, contentType, te := makeFormData(fileName, r.Header.Get("Content-Type"), r.Body) + if te != nil { + glog.V(0).Infoln("S3 protocol to raw seaweed protocol failed", te.Error()) + writeJsonError(w, r, http.StatusInternalServerError, te) + err = te + return + } + + if body != nil { + switch v := body.(type) { + case *bytes.Buffer: + r.ContentLength = int64(v.Len()) + case *bytes.Reader: + r.ContentLength = int64(v.Len()) + case *strings.Reader: + r.ContentLength = int64(v.Len()) + } + } + + r.Header.Set("Content-Type", contentType) + rc, ok := body.(io.ReadCloser) + if !ok && body != nil { + rc = ioutil.NopCloser(body) + } + r.Body = rc + return +} diff --git a/weed/server/filer_server_handlers_write_multipart.go b/weed/server/filer_server_handlers_write_multipart.go index edf9305c6..91f892b52 100644 --- a/weed/server/filer_server_handlers_write_multipart.go +++ b/weed/server/filer_server_handlers_write_multipart.go @@ -2,58 +2,14 @@ package weed_server import ( "bytes" - "fmt" - "io" "io/ioutil" - "mime/multipart" "net/http" - "net/textproto" "strings" "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/storage" ) -var quoteEscaper = strings.NewReplacer("\\", "\\\\", `"`, "\\\"") - -func escapeQuotes(s string) string { - return quoteEscaper.Replace(s) -} - -func createFormFile(writer *multipart.Writer, fieldname, filename, mime string) (io.Writer, error) { - h := make(textproto.MIMEHeader) - h.Set("Content-Disposition", - fmt.Sprintf(`form-data; name="%s"; filename="%s"`, - escapeQuotes(fieldname), escapeQuotes(filename))) - if len(mime) == 0 { - mime = "application/octet-stream" - } - h.Set("Content-Type", mime) - return writer.CreatePart(h) -} - -func makeFormData(filename, mimeType string, content io.Reader) (formData io.Reader, contentType string, err error) { - buf := new(bytes.Buffer) - writer := multipart.NewWriter(buf) - defer writer.Close() - - part, err := createFormFile(writer, "file", filename, mimeType) - if err != nil { - glog.V(0).Infoln(err) - return - } - _, err = io.Copy(part, content) - if err != nil { - glog.V(0).Infoln(err) - return - } - - formData = buf - contentType = writer.FormDataContentType() - - return -} - func (fs *FilerServer) multipartUploadAnalyzer(w http.ResponseWriter, r *http.Request, replication, collection string) (fileId, urlLocation string, err error) { //Default handle way for http multipart if r.Method == "PUT" { @@ -81,32 +37,3 @@ func (fs *FilerServer) multipartUploadAnalyzer(w http.ResponseWriter, r *http.Re } return } - -func multipartHttpBodyBuilder(w http.ResponseWriter, r *http.Request, fileName string) (err error) { - body, contentType, te := makeFormData(fileName, r.Header.Get("Content-Type"), r.Body) - if te != nil { - glog.V(0).Infoln("S3 protocol to raw seaweed protocol failed", te.Error()) - writeJsonError(w, r, http.StatusInternalServerError, te) - err = te - return - } - - if body != nil { - switch v := body.(type) { - case *bytes.Buffer: - r.ContentLength = int64(v.Len()) - case *bytes.Reader: - r.ContentLength = int64(v.Len()) - case *strings.Reader: - r.ContentLength = int64(v.Len()) - } - } - - r.Header.Set("Content-Type", contentType) - rc, ok := body.(io.ReadCloser) - if !ok && body != nil { - rc = ioutil.NopCloser(body) - } - r.Body = rc - return -} From 9b603f5ffa66a55420b366731a8244c4c87613f4 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Mon, 28 May 2018 02:24:14 -0700 Subject: [PATCH 59/83] split filer read into 2 files --- weed/server/filer_server_handlers_read.go | 61 ---------------- weed/server/filer_server_handlers_read_dir.go | 70 +++++++++++++++++++ 2 files changed, 70 insertions(+), 61 deletions(-) create mode 100644 weed/server/filer_server_handlers_read_dir.go diff --git a/weed/server/filer_server_handlers_read.go b/weed/server/filer_server_handlers_read.go index 782ed55f1..a982806e3 100644 --- a/weed/server/filer_server_handlers_read.go +++ b/weed/server/filer_server_handlers_read.go @@ -4,75 +4,14 @@ import ( "io" "net/http" "net/url" - "strconv" "strings" "github.com/chrislusf/seaweedfs/weed/filer2" "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/operation" - ui "github.com/chrislusf/seaweedfs/weed/server/filer_ui" "github.com/chrislusf/seaweedfs/weed/util" ) -// listDirectoryHandler lists directories and folers under a directory -// files are sorted by name and paginated via "lastFileName" and "limit". -// sub directories are listed on the first page, when "lastFileName" -// is empty. -func (fs *FilerServer) listDirectoryHandler(w http.ResponseWriter, r *http.Request) { - path := r.URL.Path - if strings.HasSuffix(path, "/") && len(path) > 1 { - path = path[:len(path)-1] - } - - limit, limit_err := strconv.Atoi(r.FormValue("limit")) - if limit_err != nil { - limit = 100 - } - - lastFileName := r.FormValue("lastFileName") - - entries, err := fs.filer.ListDirectoryEntries(filer2.FullPath(path), lastFileName, false, limit) - - if err != nil { - glog.V(0).Infof("listDirectory %s %s $d: %s", path, lastFileName, limit, err) - w.WriteHeader(http.StatusNotFound) - return - } - - shouldDisplayLoadMore := len(entries) == limit - if path == "/" { - path = "" - } - - if len(entries) > 0 { - lastFileName = entries[len(entries)-1].Name() - } - - glog.V(4).Infof("listDirectory %s, last file %s, limit %d: %d items", path, lastFileName, limit, len(entries)) - - args := struct { - Path string - Breadcrumbs []ui.Breadcrumb - Entries interface{} - Limit int - LastFileName string - ShouldDisplayLoadMore bool - }{ - path, - ui.ToBreadcrumb(path), - entries, - limit, - lastFileName, - shouldDisplayLoadMore, - } - - if r.Header.Get("Accept") == "application/json" { - writeJsonQuiet(w, r, http.StatusOK, args) - } else { - ui.StatusTpl.Execute(w, args) - } -} - func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request, isGetMethod bool) { path := r.URL.Path if strings.HasSuffix(path, "/") && len(path) > 1 { diff --git a/weed/server/filer_server_handlers_read_dir.go b/weed/server/filer_server_handlers_read_dir.go new file mode 100644 index 000000000..a39fed3fd --- /dev/null +++ b/weed/server/filer_server_handlers_read_dir.go @@ -0,0 +1,70 @@ +package weed_server + +import ( + "net/http" + "strconv" + "strings" + + "github.com/chrislusf/seaweedfs/weed/filer2" + "github.com/chrislusf/seaweedfs/weed/glog" + ui "github.com/chrislusf/seaweedfs/weed/server/filer_ui" +) + +// listDirectoryHandler lists directories and folers under a directory +// files are sorted by name and paginated via "lastFileName" and "limit". +// sub directories are listed on the first page, when "lastFileName" +// is empty. +func (fs *FilerServer) listDirectoryHandler(w http.ResponseWriter, r *http.Request) { + path := r.URL.Path + if strings.HasSuffix(path, "/") && len(path) > 1 { + path = path[:len(path)-1] + } + + limit, limit_err := strconv.Atoi(r.FormValue("limit")) + if limit_err != nil { + limit = 100 + } + + lastFileName := r.FormValue("lastFileName") + + entries, err := fs.filer.ListDirectoryEntries(filer2.FullPath(path), lastFileName, false, limit) + + if err != nil { + glog.V(0).Infof("listDirectory %s %s $d: %s", path, lastFileName, limit, err) + w.WriteHeader(http.StatusNotFound) + return + } + + shouldDisplayLoadMore := len(entries) == limit + if path == "/" { + path = "" + } + + if len(entries) > 0 { + lastFileName = entries[len(entries)-1].Name() + } + + glog.V(4).Infof("listDirectory %s, last file %s, limit %d: %d items", path, lastFileName, limit, len(entries)) + + args := struct { + Path string + Breadcrumbs []ui.Breadcrumb + Entries interface{} + Limit int + LastFileName string + ShouldDisplayLoadMore bool + }{ + path, + ui.ToBreadcrumb(path), + entries, + limit, + lastFileName, + shouldDisplayLoadMore, + } + + if r.Header.Get("Accept") == "application/json" { + writeJsonQuiet(w, r, http.StatusOK, args) + } else { + ui.StatusTpl.Execute(w, args) + } +} From dd5661b2976750bd142a3ccc1ac217500ca41ccc Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Mon, 28 May 2018 02:35:58 -0700 Subject: [PATCH 60/83] adding favicon to filer also --- weed/server/common.go | 13 +++++++++++++ weed/server/filer_server.go | 1 + weed/server/volume_server.go | 2 +- weed/server/volume_server_handlers.go | 4 ---- weed/server/volume_server_handlers_read.go | 12 ------------ 5 files changed, 15 insertions(+), 17 deletions(-) diff --git a/weed/server/common.go b/weed/server/common.go index 20177df0e..85d052993 100644 --- a/weed/server/common.go +++ b/weed/server/common.go @@ -12,6 +12,7 @@ import ( "time" "github.com/chrislusf/seaweedfs/weed/glog" + "github.com/chrislusf/seaweedfs/weed/images" "github.com/chrislusf/seaweedfs/weed/operation" "github.com/chrislusf/seaweedfs/weed/security" "github.com/chrislusf/seaweedfs/weed/stats" @@ -188,3 +189,15 @@ func statsMemoryHandler(w http.ResponseWriter, r *http.Request) { m["Memory"] = stats.MemStat() writeJsonQuiet(w, r, http.StatusOK, m) } + +func faviconHandler(w http.ResponseWriter, r *http.Request) { + data, err := images.Asset("favicon/favicon.ico") + if err != nil { + glog.V(2).Infoln("favicon read error:", err) + return + } + + if e := writeResponseContent("favicon.ico", "image/x-icon", bytes.NewReader(data), w, r); e != nil { + glog.V(2).Infoln("response write error:", e) + } +} diff --git a/weed/server/filer_server.go b/weed/server/filer_server.go index a1dc91650..3e175e960 100644 --- a/weed/server/filer_server.go +++ b/weed/server/filer_server.go @@ -53,6 +53,7 @@ func NewFilerServer(defaultMux, readonlyMux *http.ServeMux, ip string, port int, fs.filer.LoadConfiguration() defaultMux.HandleFunc("/admin/register", fs.registerHandler) + defaultMux.HandleFunc("/favicon.ico", faviconHandler) defaultMux.HandleFunc("/", fs.filerHandler) if defaultMux != readonlyMux { readonlyMux.HandleFunc("/", fs.readonlyFilerHandler) diff --git a/weed/server/volume_server.go b/weed/server/volume_server.go index b0620de0b..0b7e09c59 100644 --- a/weed/server/volume_server.go +++ b/weed/server/volume_server.go @@ -67,7 +67,7 @@ func NewVolumeServer(adminMux, publicMux *http.ServeMux, ip string, adminMux.HandleFunc("/", vs.privateStoreHandler) if publicMux != adminMux { // separated admin and public port - publicMux.HandleFunc("/favicon.ico", vs.faviconHandler) + publicMux.HandleFunc("/favicon.ico", faviconHandler) publicMux.HandleFunc("/", vs.publicReadOnlyHandler) } diff --git a/weed/server/volume_server_handlers.go b/weed/server/volume_server_handlers.go index 2d6fe7849..6ef79dcdb 100644 --- a/weed/server/volume_server_handlers.go +++ b/weed/server/volume_server_handlers.go @@ -51,7 +51,3 @@ func (vs *VolumeServer) publicReadOnlyHandler(w http.ResponseWriter, r *http.Req vs.GetOrHeadHandler(w, r) } } - -func (vs *VolumeServer) faviconHandler(w http.ResponseWriter, r *http.Request) { - vs.FaviconHandler(w, r) -} diff --git a/weed/server/volume_server_handlers_read.go b/weed/server/volume_server_handlers_read.go index 9b0fee4eb..a90d4c0e2 100644 --- a/weed/server/volume_server_handlers_read.go +++ b/weed/server/volume_server_handlers_read.go @@ -151,18 +151,6 @@ func (vs *VolumeServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request) } } -func (vs *VolumeServer) FaviconHandler(w http.ResponseWriter, r *http.Request) { - data, err := images.Asset("favicon/favicon.ico") - if err != nil { - glog.V(2).Infoln("favicon read error:", err) - return - } - - if e := writeResponseContent("favicon.ico", "image/x-icon", bytes.NewReader(data), w, r); e != nil { - glog.V(2).Infoln("response write error:", e) - } -} - func (vs *VolumeServer) tryHandleChunkedFile(n *storage.Needle, fileName string, w http.ResponseWriter, r *http.Request) (processed bool) { if !n.IsChunkedManifest() || r.URL.Query().Get("cm") == "false" { return false From 07e0d13d2d13f53a6540e836189741186aaf060e Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Mon, 28 May 2018 05:39:12 -0700 Subject: [PATCH 61/83] filer support reading multiple chunks, with range support --- weed/filer2/filechunks.go | 2 +- weed/filer2/filechunks_test.go | 2 +- weed/filesys/filehandle.go | 2 +- weed/server/filer_server_handlers_read.go | 165 +++++++++++++++++++++- weed/util/http_util.go | 34 +++++ 5 files changed, 198 insertions(+), 7 deletions(-) diff --git a/weed/filer2/filechunks.go b/weed/filer2/filechunks.go index c4c77d270..0ac7bb43b 100644 --- a/weed/filer2/filechunks.go +++ b/weed/filer2/filechunks.go @@ -58,7 +58,7 @@ type ChunkView struct { LogicOffset int64 } -func ReadFromChunks(chunks []*filer_pb.FileChunk, offset int64, size int) (views []*ChunkView) { +func ViewFromChunks(chunks []*filer_pb.FileChunk, offset int64, size int) (views []*ChunkView) { visibles := nonOverlappingVisibleIntervals(chunks) diff --git a/weed/filer2/filechunks_test.go b/weed/filer2/filechunks_test.go index 134bb9c19..c3f7e0504 100644 --- a/weed/filer2/filechunks_test.go +++ b/weed/filer2/filechunks_test.go @@ -267,7 +267,7 @@ func TestChunksReading(t *testing.T) { for i, testcase := range testcases { log.Printf("++++++++++ read test case %d ++++++++++++++++++++", i) - chunks := ReadFromChunks(testcase.Chunks, testcase.Offset, testcase.Size) + chunks := ViewFromChunks(testcase.Chunks, testcase.Offset, testcase.Size) for x, chunk := range chunks { log.Printf("read case %d, chunk %d, offset=%d, size=%d, fileId=%s", i, x, chunk.Offset, chunk.Size, chunk.FileId) diff --git a/weed/filesys/filehandle.go b/weed/filesys/filehandle.go index 81aca42a4..d3c8ec796 100644 --- a/weed/filesys/filehandle.go +++ b/weed/filesys/filehandle.go @@ -50,7 +50,7 @@ func (fh *FileHandle) Read(ctx context.Context, req *fuse.ReadRequest, resp *fus buff := make([]byte, req.Size) - chunkViews := filer2.ReadFromChunks(fh.f.Chunks, req.Offset, req.Size) + chunkViews := filer2.ViewFromChunks(fh.f.Chunks, req.Offset, req.Size) var vids []string for _, chunkView := range chunkViews { diff --git a/weed/server/filer_server_handlers_read.go b/weed/server/filer_server_handlers_read.go index a982806e3..08430716c 100644 --- a/weed/server/filer_server_handlers_read.go +++ b/weed/server/filer_server_handlers_read.go @@ -10,6 +10,10 @@ import ( "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/operation" "github.com/chrislusf/seaweedfs/weed/util" + "strconv" + "mime/multipart" + "mime" + "path" ) func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request, isGetMethod bool) { @@ -40,20 +44,37 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request, return } - // FIXME pick the right fid + w.Header().Set("Accept-Ranges", "bytes") + if r.Method == "HEAD" { + w.Header().Set("Content-Length", strconv.FormatInt(int64(filer2.TotalSize(entry.Chunks)), 10)) + return + } + + if len(entry.Chunks) == 1 { + fs.handleSingleChunk(w, r, entry) + return + } + + fs.handleMultipleChunks(w, r, entry) + +} + +func (fs *FilerServer) handleSingleChunk(w http.ResponseWriter, r *http.Request, entry *filer2.Entry) { + fileId := entry.Chunks[0].FileId - urlLocation, err := operation.LookupFileId(fs.getMasterNode(), fileId) + urlString, err := operation.LookupFileId(fs.getMasterNode(), fileId) if err != nil { - glog.V(1).Infoln("operation LookupFileId %s failed, err is %s", fileId, err.Error()) + glog.V(1).Infof("operation LookupFileId %s failed, err: %v", fileId, err) w.WriteHeader(http.StatusNotFound) return } - urlString := urlLocation + if fs.redirectOnRead { http.Redirect(w, r, urlString, http.StatusFound) return } + u, _ := url.Parse(urlString) q := u.Query() for key, values := range r.URL.Query() { @@ -86,5 +107,141 @@ func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request, } w.WriteHeader(resp.StatusCode) io.Copy(w, resp.Body) +} + +func (fs *FilerServer) handleMultipleChunks(w http.ResponseWriter, r *http.Request, entry *filer2.Entry) { + + mimeType := "" + if ext := path.Ext(entry.Name()); ext != "" { + mimeType = mime.TypeByExtension(ext) + } + if mimeType != "" { + w.Header().Set("Content-Type", mimeType) + } + + println("mime type:", mimeType) + + totalSize := int64(filer2.TotalSize(entry.Chunks)) + + rangeReq := r.Header.Get("Range") + + if rangeReq == "" { + w.Header().Set("Content-Length", strconv.FormatInt(totalSize, 10)) + if err := fs.writeContent(w, entry, 0, int(totalSize)); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + return + } + + //the rest is dealing with partial content request + //mostly copy from src/pkg/net/http/fs.go + ranges, err := parseRange(rangeReq, totalSize) + if err != nil { + http.Error(w, err.Error(), http.StatusRequestedRangeNotSatisfiable) + return + } + if sumRangesSize(ranges) > totalSize { + // The total number of bytes in all the ranges + // is larger than the size of the file by + // itself, so this is probably an attack, or a + // dumb client. Ignore the range request. + return + } + if len(ranges) == 0 { + return + } + if len(ranges) == 1 { + // RFC 2616, Section 14.16: + // "When an HTTP message includes the content of a single + // range (for example, a response to a request for a + // single range, or to a request for a set of ranges + // that overlap without any holes), this content is + // transmitted with a Content-Range header, and a + // Content-Length header showing the number of bytes + // actually transferred. + // ... + // A response to a request for a single range MUST NOT + // be sent using the multipart/byteranges media type." + ra := ranges[0] + w.Header().Set("Content-Length", strconv.FormatInt(ra.length, 10)) + w.Header().Set("Content-Range", ra.contentRange(totalSize)) + w.WriteHeader(http.StatusPartialContent) + + err = fs.writeContent(w, entry, ra.start, int(ra.length)) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + return + } + + // process multiple ranges + for _, ra := range ranges { + if ra.start > totalSize { + http.Error(w, "Out of Range", http.StatusRequestedRangeNotSatisfiable) + return + } + } + sendSize := rangesMIMESize(ranges, mimeType, totalSize) + pr, pw := io.Pipe() + mw := multipart.NewWriter(pw) + w.Header().Set("Content-Type", "multipart/byteranges; boundary="+mw.Boundary()) + sendContent := pr + defer pr.Close() // cause writing goroutine to fail and exit if CopyN doesn't finish. + go func() { + for _, ra := range ranges { + part, e := mw.CreatePart(ra.mimeHeader(mimeType, totalSize)) + if e != nil { + pw.CloseWithError(e) + return + } + if e = fs.writeContent(part, entry, ra.start, int(ra.length)); e != nil { + pw.CloseWithError(e) + return + } + } + mw.Close() + pw.Close() + }() + if w.Header().Get("Content-Encoding") == "" { + w.Header().Set("Content-Length", strconv.FormatInt(sendSize, 10)) + } + w.WriteHeader(http.StatusPartialContent) + if _, err := io.CopyN(w, sendContent, sendSize); err != nil { + http.Error(w, "Internal Error", http.StatusInternalServerError) + return + } + +} + +func (fs *FilerServer) writeContent(w io.Writer, entry *filer2.Entry, offset int64, size int) error { + + chunkViews := filer2.ViewFromChunks(entry.Chunks, offset, size) + + fileId2Url := make(map[string]string) + + for _, chunkView := range chunkViews { + + urlString, err := operation.LookupFileId(fs.getMasterNode(), chunkView.FileId) + if err != nil { + glog.V(1).Infof("operation LookupFileId %s failed, err: %v", chunkView.FileId, err) + return err + } + fileId2Url[chunkView.FileId] = urlString + } + + for _, chunkView := range chunkViews { + urlString := fileId2Url[chunkView.FileId] + _, err := util.ReadUrlAsStream(urlString, chunkView.Offset, int(chunkView.Size), func(data []byte) { + w.Write(data) + }) + if err != nil { + glog.V(1).Infof("read %s failed, err: %v", chunkView.FileId, err) + return err + } + } + + return nil } diff --git a/weed/util/http_util.go b/weed/util/http_util.go index 579abaac0..51bedcdfd 100644 --- a/weed/util/http_util.go +++ b/weed/util/http_util.go @@ -215,3 +215,37 @@ func ReadUrl(fileUrl string, offset int64, size int, buf []byte) (n int64, e err } } + +func ReadUrlAsStream(fileUrl string, offset int64, size int, fn func(data []byte)) (n int64, e error) { + + req, _ := http.NewRequest("GET", fileUrl, nil) + req.Header.Add("Range", fmt.Sprintf("bytes=%d-%d", offset, offset+int64(size))) + + r, err := client.Do(req) + if err != nil { + return 0, err + } + defer r.Body.Close() + if r.StatusCode >= 400 { + return 0, fmt.Errorf("%s: %s", fileUrl, r.Status) + } + + var m int + buf := make([]byte, 64*1024) + + for { + m, err = r.Body.Read(buf) + if m == 0 { + return + } + fn(buf[:m]) + n += int64(m) + if err == io.EOF { + return n, nil + } + if e != nil { + return n, e + } + } + +} From d0b238d2db81093b7f02141d4f70b446fcd0b2cd Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Mon, 28 May 2018 12:30:17 -0700 Subject: [PATCH 62/83] cache local writes before flushing to volume server --- weed/filesys/dirty_page.go | 152 +++++++++++++++++++++++++++++++++++++ weed/filesys/file.go | 11 +-- weed/filesys/filehandle.go | 75 ++++++------------ 3 files changed, 182 insertions(+), 56 deletions(-) create mode 100644 weed/filesys/dirty_page.go diff --git a/weed/filesys/dirty_page.go b/weed/filesys/dirty_page.go new file mode 100644 index 000000000..bfb73f3b0 --- /dev/null +++ b/weed/filesys/dirty_page.go @@ -0,0 +1,152 @@ +package filesys + +import ( + "sync" + "sort" + "fmt" + "bytes" + "io" + "time" + "context" + + "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" + "github.com/chrislusf/seaweedfs/weed/operation" + "github.com/chrislusf/seaweedfs/weed/glog" +) + +type DirtyPage struct { + Offset int64 + Data []byte +} + +type ContinuousDirtyPages struct { + sync.Mutex + + pages []*DirtyPage + f *File +} + +func (pages *ContinuousDirtyPages) AddPage(ctx context.Context, offset int64, data []byte) (chunk *filer_pb.FileChunk, err error) { + pages.Lock() + defer pages.Unlock() + + isPerfectAppend := len(pages.pages) == 0 + if len(pages.pages) > 0 { + lastPage := pages.pages[len(pages.pages)-1] + if lastPage.Offset+int64(len(lastPage.Data)) == offset { + // write continuous pages + glog.V(3).Infof("%s/%s append [%d,%d)", pages.f.dir.Path, pages.f.Name, offset, offset+int64(len(data))) + isPerfectAppend = true + } + } + + isPerfectReplace := false + for _, page := range pages.pages { + if page.Offset == offset && len(page.Data) == len(data) { + // perfect replace + glog.V(3).Infof("%s/%s replace [%d,%d)", pages.f.dir.Path, pages.f.Name, offset, offset+int64(len(data))) + page.Data = data + isPerfectReplace = true + } + } + + if isPerfectReplace { + return nil, nil + } + + if isPerfectAppend { + pages.pages = append(pages.pages, &DirtyPage{ + Offset: offset, + Data: data, + }) + return nil, nil + } + + chunk, err = pages.saveToStorage(ctx) + + glog.V(3).Infof("%s/%s saved [%d,%d)", pages.f.dir.Path, pages.f.Name, chunk.Offset, chunk.Offset+int64(chunk.Size)) + + pages.pages = []*DirtyPage{&DirtyPage{ + Offset: offset, + Data: data, + }} + + return +} + +func (pages *ContinuousDirtyPages) FlushToStorage(ctx context.Context) (chunk *filer_pb.FileChunk, err error) { + + pages.Lock() + defer pages.Unlock() + + if chunk, err = pages.saveToStorage(ctx); err == nil { + pages.pages = nil + } + return +} + +func (pages *ContinuousDirtyPages) totalSize() (total int64) { + for _, page := range pages.pages { + total += int64(len(page.Data)) + } + return +} + +func (pages *ContinuousDirtyPages) saveToStorage(ctx context.Context) (*filer_pb.FileChunk, error) { + + if len(pages.pages) == 0 { + return nil, nil + } + + sort.Slice(pages.pages, func(i, j int) bool { + return pages.pages[i].Offset < pages.pages[j].Offset + }) + + var fileId, host string + + if err := pages.f.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error { + + request := &filer_pb.AssignVolumeRequest{ + Count: 1, + Replication: "000", + Collection: "", + } + + resp, err := client.AssignVolume(ctx, request) + if err != nil { + glog.V(0).Infof("assign volume failure %v: %v", request, err) + return err + } + + fileId, host = resp.FileId, resp.Url + + return nil + }); err != nil { + return nil, fmt.Errorf("filer assign volume: %v", err) + } + + var readers []io.Reader + for _, page := range pages.pages { + readers = append(readers, bytes.NewReader(page.Data)) + } + + fileUrl := fmt.Sprintf("http://%s/%s", host, fileId) + bufReader := io.MultiReader(readers...) + uploadResult, err := operation.Upload(fileUrl, pages.f.Name, bufReader, false, "application/octet-stream", nil, "") + if err != nil { + glog.V(0).Infof("upload data %v to %s: %v", pages.f.Name, fileUrl, err) + return nil, fmt.Errorf("upload data: %v", err) + } + if uploadResult.Error != "" { + glog.V(0).Infof("upload failure %v to %s: %v", pages.f.Name, fileUrl, err) + return nil, fmt.Errorf("upload result: %v", uploadResult.Error) + } + + return &filer_pb.FileChunk{ + FileId: fileId, + Offset: pages.pages[0].Offset, + Size: uint64(pages.totalSize()), + Mtime: time.Now().UnixNano(), + }, nil + +} diff --git a/weed/filesys/file.go b/weed/filesys/file.go index 7ea14cc49..255fe4af0 100644 --- a/weed/filesys/file.go +++ b/weed/filesys/file.go @@ -84,11 +84,12 @@ func (file *File) Open(ctx context.Context, req *fuse.OpenRequest, resp *fuse.Op file.isOpen = true return &FileHandle{ - f: file, - RequestId: req.Header.ID, - NodeId: req.Header.Node, - Uid: req.Uid, - Gid: req.Gid, + f: file, + dirtyPages: &ContinuousDirtyPages{f: file}, + RequestId: req.Header.ID, + NodeId: req.Header.Node, + Uid: req.Uid, + Gid: req.Gid, }, nil } diff --git a/weed/filesys/filehandle.go b/weed/filesys/filehandle.go index d3c8ec796..52df2824c 100644 --- a/weed/filesys/filehandle.go +++ b/weed/filesys/filehandle.go @@ -3,22 +3,20 @@ package filesys import ( "bazil.org/fuse" "bazil.org/fuse/fs" - "bytes" "context" "fmt" "github.com/chrislusf/seaweedfs/weed/filer2" "github.com/chrislusf/seaweedfs/weed/glog" - "github.com/chrislusf/seaweedfs/weed/operation" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "github.com/chrislusf/seaweedfs/weed/util" "strings" "sync" - "time" ) type FileHandle struct { // cache file has been written to - dirty bool + dirtyPages *ContinuousDirtyPages + dirtyMetadata bool cachePath string @@ -128,55 +126,20 @@ func (fh *FileHandle) Write(ctx context.Context, req *fuse.WriteRequest, resp *f // write the request to volume servers - glog.V(3).Infof("%+v/%v write fh: %+v", fh.f.dir.Path, fh.f.Name, req) + glog.V(3).Infof("%+v/%v write fh: [%d,%d)", fh.f.dir.Path, fh.f.Name, req.Offset, req.Offset+int64(len(req.Data))) - var fileId, host string - - if err := fh.f.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error { - - request := &filer_pb.AssignVolumeRequest{ - Count: 1, - Replication: "000", - Collection: "", - } - - resp, err := client.AssignVolume(ctx, request) - if err != nil { - glog.V(0).Infof("assign volume failure %v: %v", request, err) - return err - } - - fileId, host = resp.FileId, resp.Url - - return nil - }); err != nil { - return fmt.Errorf("filer assign volume: %v", err) - } - - fileUrl := fmt.Sprintf("http://%s/%s", host, fileId) - bufReader := bytes.NewReader(req.Data) - uploadResult, err := operation.Upload(fileUrl, fh.f.Name, bufReader, false, "application/octet-stream", nil, "") + chunk, err := fh.dirtyPages.AddPage(ctx, req.Offset, req.Data) if err != nil { - glog.V(0).Infof("upload data %v to %s: %v", req, fileUrl, err) - return fmt.Errorf("upload data: %v", err) - } - if uploadResult.Error != "" { - glog.V(0).Infof("upload failure %v to %s: %v", req, fileUrl, err) - return fmt.Errorf("upload result: %v", uploadResult.Error) + return fmt.Errorf("write %s/%s at [%d,%d): %v", fh.f.dir.Path, fh.f.Name, req.Offset, req.Offset+int64(len(req.Data)), err) } - resp.Size = int(uploadResult.Size) - - fh.f.Chunks = append(fh.f.Chunks, &filer_pb.FileChunk{ - FileId: fileId, - Offset: req.Offset, - Size: uint64(uploadResult.Size), - Mtime: time.Now().UnixNano(), - }) - - glog.V(1).Infof("uploaded %s/%s to: %v, [%d,%d)", fh.f.dir.Path, fh.f.Name, fileUrl, req.Offset, req.Offset+int64(resp.Size)) + resp.Size = len(req.Data) - fh.dirty = true + if chunk != nil { + fh.f.Chunks = append(fh.f.Chunks, chunk) + glog.V(1).Infof("uploaded %s/%s to %s [%d,%d)", fh.f.dir.Path, fh.f.Name, chunk.FileId, chunk.Offset, chunk.Offset+int64(chunk.Size)) + fh.dirtyMetadata = true + } return nil } @@ -197,7 +160,17 @@ func (fh *FileHandle) Flush(ctx context.Context, req *fuse.FlushRequest) error { // send the data to the OS glog.V(3).Infof("%s/%s fh flush %v", fh.f.dir.Path, fh.f.Name, req) - if !fh.dirty { + chunk, err := fh.dirtyPages.FlushToStorage(ctx) + if err != nil { + glog.V(0).Infof("flush %s/%s to %s [%d,%d): %v", fh.f.dir.Path, fh.f.Name, chunk.FileId, chunk.Offset, chunk.Offset+int64(chunk.Size), err) + return fmt.Errorf("flush %s/%s to %s [%d,%d): %v", fh.f.dir.Path, fh.f.Name, chunk.FileId, chunk.Offset, chunk.Offset+int64(chunk.Size), err) + } + if chunk != nil { + fh.f.Chunks = append(fh.f.Chunks, chunk) + fh.dirtyMetadata = true + } + + if !fh.dirtyMetadata { return nil } @@ -206,7 +179,7 @@ func (fh *FileHandle) Flush(ctx context.Context, req *fuse.FlushRequest) error { return nil } - err := fh.f.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error { + err = fh.f.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error { request := &filer_pb.UpdateEntryRequest{ Directory: fh.f.dir.Path, @@ -229,7 +202,7 @@ func (fh *FileHandle) Flush(ctx context.Context, req *fuse.FlushRequest) error { }) if err == nil { - fh.dirty = false + fh.dirtyMetadata = false } return err From 8ab7dd9d08dbf326046bfcf0c1fac5d171600a7d Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Mon, 28 May 2018 13:24:48 -0700 Subject: [PATCH 63/83] weed mount add options for collection and replication --- weed/command/mount.go | 8 ++++++-- weed/command/mount_std.go | 3 ++- weed/filesys/dir.go | 11 ++++++----- weed/filesys/dirty_page.go | 4 ++-- weed/filesys/wfs.go | 8 ++++++-- 5 files changed, 22 insertions(+), 12 deletions(-) diff --git a/weed/command/mount.go b/weed/command/mount.go index 746e4b92e..151c6bedf 100644 --- a/weed/command/mount.go +++ b/weed/command/mount.go @@ -1,8 +1,10 @@ package command type MountOptions struct { - filer *string - dir *string + filer *string + dir *string + collection *string + replication *string } var ( @@ -14,6 +16,8 @@ func init() { cmdMount.IsDebug = cmdMount.Flag.Bool("debug", false, "verbose debug information") mountOptions.filer = cmdMount.Flag.String("filer", "localhost:8888", "weed filer location") mountOptions.dir = cmdMount.Flag.String("dir", ".", "mount weed filer to this directory") + mountOptions.collection = cmdMount.Flag.String("collection", "", "collection to create the files") + mountOptions.replication = cmdMount.Flag.String("replication", "000", "replication to create to files") } var cmdMount = &Command{ diff --git a/weed/command/mount_std.go b/weed/command/mount_std.go index b3f038cc2..fcf663e1c 100644 --- a/weed/command/mount_std.go +++ b/weed/command/mount_std.go @@ -47,7 +47,8 @@ func runMount(cmd *Command, args []string) bool { c.Close() }) - err = fs.Serve(c, filesys.NewSeaweedFileSystem(*mountOptions.filer)) + err = fs.Serve(c, filesys.NewSeaweedFileSystem( + *mountOptions.filer, *mountOptions.collection, *mountOptions.replication)) if err != nil { fuse.Unmount(*mountOptions.dir) } diff --git a/weed/filesys/dir.go b/weed/filesys/dir.go index e53c4cfaf..b90e428ab 100644 --- a/weed/filesys/dir.go +++ b/weed/filesys/dir.go @@ -123,11 +123,12 @@ func (dir *Dir) Create(ctx context.Context, req *fuse.CreateRequest, dir.NodeMap[req.Name] = file file.isOpen = true return file, &FileHandle{ - f: file, - RequestId: req.Header.ID, - NodeId: req.Header.Node, - Uid: req.Uid, - Gid: req.Gid, + f: file, + dirtyPages: &ContinuousDirtyPages{f: file}, + RequestId: req.Header.ID, + NodeId: req.Header.Node, + Uid: req.Uid, + Gid: req.Gid, }, nil } diff --git a/weed/filesys/dirty_page.go b/weed/filesys/dirty_page.go index bfb73f3b0..da442d6c6 100644 --- a/weed/filesys/dirty_page.go +++ b/weed/filesys/dirty_page.go @@ -108,8 +108,8 @@ func (pages *ContinuousDirtyPages) saveToStorage(ctx context.Context) (*filer_pb request := &filer_pb.AssignVolumeRequest{ Count: 1, - Replication: "000", - Collection: "", + Replication: pages.f.wfs.replication, + Collection: pages.f.wfs.collection, } resp, err := client.AssignVolume(ctx, request) diff --git a/weed/filesys/wfs.go b/weed/filesys/wfs.go index 1b843e2d7..b9cb0210b 100644 --- a/weed/filesys/wfs.go +++ b/weed/filesys/wfs.go @@ -11,12 +11,16 @@ import ( type WFS struct { filer string listDirectoryEntriesCache *ccache.Cache + collection string + replication string } -func NewSeaweedFileSystem(filer string) *WFS { +func NewSeaweedFileSystem(filer string, collection string, replication string) *WFS { return &WFS{ - filer: filer, + filer: filer, listDirectoryEntriesCache: ccache.New(ccache.Configure().MaxSize(6000).ItemsToPrune(100)), + collection: collection, + replication: replication, } } From 5c4480ec6c941e16efd4157ee98e67d21b5859fb Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Mon, 28 May 2018 13:42:25 -0700 Subject: [PATCH 64/83] add mountOptions.chunkSizeLimitMB, remove cmdMount.IsDebug --- weed/command/mount.go | 11 ++++++----- weed/command/mount_std.go | 2 +- weed/filesys/dirty_page.go | 8 +++++++- weed/filesys/wfs.go | 4 +++- 4 files changed, 17 insertions(+), 8 deletions(-) diff --git a/weed/command/mount.go b/weed/command/mount.go index 151c6bedf..f0514b93a 100644 --- a/weed/command/mount.go +++ b/weed/command/mount.go @@ -1,10 +1,11 @@ package command type MountOptions struct { - filer *string - dir *string - collection *string - replication *string + filer *string + dir *string + collection *string + replication *string + chunkSizeLimitMB *int } var ( @@ -13,11 +14,11 @@ var ( func init() { cmdMount.Run = runMount // break init cycle - cmdMount.IsDebug = cmdMount.Flag.Bool("debug", false, "verbose debug information") mountOptions.filer = cmdMount.Flag.String("filer", "localhost:8888", "weed filer location") mountOptions.dir = cmdMount.Flag.String("dir", ".", "mount weed filer to this directory") mountOptions.collection = cmdMount.Flag.String("collection", "", "collection to create the files") mountOptions.replication = cmdMount.Flag.String("replication", "000", "replication to create to files") + mountOptions.chunkSizeLimitMB = cmdMount.Flag.Int("chunkSizeLimitMB", 0, "if set, limit the chunk size in MB") } var cmdMount = &Command{ diff --git a/weed/command/mount_std.go b/weed/command/mount_std.go index fcf663e1c..7e7a35f3d 100644 --- a/weed/command/mount_std.go +++ b/weed/command/mount_std.go @@ -48,7 +48,7 @@ func runMount(cmd *Command, args []string) bool { }) err = fs.Serve(c, filesys.NewSeaweedFileSystem( - *mountOptions.filer, *mountOptions.collection, *mountOptions.replication)) + *mountOptions.filer, *mountOptions.collection, *mountOptions.replication, *mountOptions.chunkSizeLimitMB)) if err != nil { fuse.Unmount(*mountOptions.dir) } diff --git a/weed/filesys/dirty_page.go b/weed/filesys/dirty_page.go index da442d6c6..a78fdb47d 100644 --- a/weed/filesys/dirty_page.go +++ b/weed/filesys/dirty_page.go @@ -59,7 +59,13 @@ func (pages *ContinuousDirtyPages) AddPage(ctx context.Context, offset int64, da Offset: offset, Data: data, }) - return nil, nil + + if pages.totalSize() >= pages.f.wfs.chunkSizeLimit { + chunk, err = pages.saveToStorage(ctx) + pages.pages = nil + glog.V(3).Infof("%s/%s add split [%d,%d)", pages.f.dir.Path, pages.f.Name, chunk.Offset, chunk.Offset+int64(chunk.Size)) + } + return } chunk, err = pages.saveToStorage(ctx) diff --git a/weed/filesys/wfs.go b/weed/filesys/wfs.go index b9cb0210b..4b9e20b95 100644 --- a/weed/filesys/wfs.go +++ b/weed/filesys/wfs.go @@ -13,14 +13,16 @@ type WFS struct { listDirectoryEntriesCache *ccache.Cache collection string replication string + chunkSizeLimit int64 } -func NewSeaweedFileSystem(filer string, collection string, replication string) *WFS { +func NewSeaweedFileSystem(filer string, collection string, replication string, chunkSizeLimitMB int) *WFS { return &WFS{ filer: filer, listDirectoryEntriesCache: ccache.New(ccache.Configure().MaxSize(6000).ItemsToPrune(100)), collection: collection, replication: replication, + chunkSizeLimit: int64(chunkSizeLimitMB) * 1024 * 1024, } } From be0e88a6068e61f17330502e3128723f0b2640d4 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Mon, 28 May 2018 13:44:27 -0700 Subject: [PATCH 65/83] fix chunk size limit for default 0 --- weed/filesys/dirty_page.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/weed/filesys/dirty_page.go b/weed/filesys/dirty_page.go index a78fdb47d..e4e743166 100644 --- a/weed/filesys/dirty_page.go +++ b/weed/filesys/dirty_page.go @@ -60,7 +60,7 @@ func (pages *ContinuousDirtyPages) AddPage(ctx context.Context, offset int64, da Data: data, }) - if pages.totalSize() >= pages.f.wfs.chunkSizeLimit { + if pages.f.wfs.chunkSizeLimit > 0 && pages.totalSize() >= pages.f.wfs.chunkSizeLimit { chunk, err = pages.saveToStorage(ctx) pages.pages = nil glog.V(3).Infof("%s/%s add split [%d,%d)", pages.f.dir.Path, pages.f.Name, chunk.Offset, chunk.Offset+int64(chunk.Size)) From 74332e1a619b3be449994e38f26247e4cdf883ef Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Mon, 28 May 2018 14:32:16 -0700 Subject: [PATCH 66/83] minor --- weed/filer2/filechunks_test.go | 20 ++++++++++++++++++++ weed/filesys/dirty_page.go | 10 ++++------ 2 files changed, 24 insertions(+), 6 deletions(-) diff --git a/weed/filer2/filechunks_test.go b/weed/filer2/filechunks_test.go index c3f7e0504..90aa3df36 100644 --- a/weed/filer2/filechunks_test.go +++ b/weed/filer2/filechunks_test.go @@ -117,6 +117,24 @@ func TestIntervalMerging(t *testing.T) { {start: 0, stop: 100, fileId: "abc"}, }, }, + // case 7: real updates + { + Chunks: []*filer_pb.FileChunk{ + {Offset: 0, Size: 2097152, FileId: "7,0294cbb9892b", Mtime: 123}, + {Offset: 0, Size: 3145728, FileId: "3,029565bf3092", Mtime: 130}, + {Offset: 2097152, Size: 3145728, FileId: "6,029632f47ae2", Mtime: 140}, + {Offset: 5242880, Size: 3145728, FileId: "2,029734c5aa10", Mtime: 150}, + {Offset: 8388608, Size: 3145728, FileId: "5,02982f80de50", Mtime: 160}, + {Offset: 11534336, Size: 2842193, FileId: "7,0299ad723803", Mtime: 170}, + }, + Expected: []*visibleInterval{ + {start: 0, stop: 2097152, fileId: "3,029565bf3092"}, + {start: 2097152, stop: 5242880, fileId: "6,029632f47ae2"}, + {start: 5242880, stop: 8388608, fileId: "2,029734c5aa10"}, + {start: 8388608, stop: 11534336, fileId: "5,02982f80de50"}, + {start: 11534336, stop: 14376529, fileId: "7,0299ad723803"}, + }, + }, } for i, testcase := range testcases { @@ -125,6 +143,8 @@ func TestIntervalMerging(t *testing.T) { for x, interval := range intervals { log.Printf("test case %d, interval %d, start=%d, stop=%d, fileId=%s", i, x, interval.start, interval.stop, interval.fileId) + } + for x, interval := range intervals { if interval.start != testcase.Expected[x].start { t.Fatalf("failed on test case %d, interval %d, start %d, expect %d", i, x, interval.start, testcase.Expected[x].start) diff --git a/weed/filesys/dirty_page.go b/weed/filesys/dirty_page.go index e4e743166..584db809d 100644 --- a/weed/filesys/dirty_page.go +++ b/weed/filesys/dirty_page.go @@ -2,7 +2,6 @@ package filesys import ( "sync" - "sort" "fmt" "bytes" "io" @@ -63,7 +62,7 @@ func (pages *ContinuousDirtyPages) AddPage(ctx context.Context, offset int64, da if pages.f.wfs.chunkSizeLimit > 0 && pages.totalSize() >= pages.f.wfs.chunkSizeLimit { chunk, err = pages.saveToStorage(ctx) pages.pages = nil - glog.V(3).Infof("%s/%s add split [%d,%d)", pages.f.dir.Path, pages.f.Name, chunk.Offset, chunk.Offset+int64(chunk.Size)) + glog.V(3).Infof("%s/%s over size limit [%d,%d)", pages.f.dir.Path, pages.f.Name, chunk.Offset, chunk.Offset+int64(chunk.Size)) } return } @@ -87,6 +86,9 @@ func (pages *ContinuousDirtyPages) FlushToStorage(ctx context.Context) (chunk *f if chunk, err = pages.saveToStorage(ctx); err == nil { pages.pages = nil + if chunk != nil { + glog.V(3).Infof("%s/%s flush [%d,%d)", pages.f.dir.Path, pages.f.Name, chunk.Offset, chunk.Offset+int64(chunk.Size)) + } } return } @@ -104,10 +106,6 @@ func (pages *ContinuousDirtyPages) saveToStorage(ctx context.Context) (*filer_pb return nil, nil } - sort.Slice(pages.pages, func(i, j int) bool { - return pages.pages[i].Offset < pages.pages[j].Offset - }) - var fileId, host string if err := pages.f.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error { From c4b92e17d033df1e348d3ab83c1ba5603e473a4f Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Mon, 28 May 2018 22:45:52 -0700 Subject: [PATCH 67/83] fix isPerfectAppend --- weed/filesys/dirty_page.go | 41 ++++++++++++++++++++++++++++---------- weed/filesys/filehandle.go | 14 ++++++------- 2 files changed, 37 insertions(+), 18 deletions(-) diff --git a/weed/filesys/dirty_page.go b/weed/filesys/dirty_page.go index 584db809d..04e05b440 100644 --- a/weed/filesys/dirty_page.go +++ b/weed/filesys/dirty_page.go @@ -29,21 +29,29 @@ func (pages *ContinuousDirtyPages) AddPage(ctx context.Context, offset int64, da pages.Lock() defer pages.Unlock() - isPerfectAppend := len(pages.pages) == 0 + isPerfectOverwrite := false + isPerfectAppend := false if len(pages.pages) > 0 { lastPage := pages.pages[len(pages.pages)-1] if lastPage.Offset+int64(len(lastPage.Data)) == offset { // write continuous pages - glog.V(3).Infof("%s/%s append [%d,%d)", pages.f.dir.Path, pages.f.Name, offset, offset+int64(len(data))) + glog.V(4).Infof("%s/%s append [%d,%d)", pages.f.dir.Path, pages.f.Name, offset, offset+int64(len(data))) isPerfectAppend = true } + if pages.pages[0].Offset == offset && pages.totalSize() == int64(len(data)) { + glog.V(4).Infof("%s/%s overwrite [%d,%d)", pages.f.dir.Path, pages.f.Name, offset, offset+int64(len(data))) + isPerfectOverwrite = true + } + } else { + glog.V(4).Infof("%s/%s append [%d,%d)", pages.f.dir.Path, pages.f.Name, offset, offset+int64(len(data))) + isPerfectAppend = true } isPerfectReplace := false for _, page := range pages.pages { if page.Offset == offset && len(page.Data) == len(data) { // perfect replace - glog.V(3).Infof("%s/%s replace [%d,%d)", pages.f.dir.Path, pages.f.Name, offset, offset+int64(len(data))) + glog.V(4).Infof("%s/%s replace [%d,%d)", pages.f.dir.Path, pages.f.Name, offset, offset+int64(len(data))) page.Data = data isPerfectReplace = true } @@ -53,23 +61,34 @@ func (pages *ContinuousDirtyPages) AddPage(ctx context.Context, offset int64, da return nil, nil } - if isPerfectAppend { - pages.pages = append(pages.pages, &DirtyPage{ - Offset: offset, - Data: data, - }) + if isPerfectAppend || isPerfectOverwrite { + if isPerfectAppend { + glog.V(4).Infof("%s/%s append2 [%d,%d)", pages.f.dir.Path, pages.f.Name, offset, offset+int64(len(data))) + pages.pages = append(pages.pages, &DirtyPage{ + Offset: offset, + Data: data, + }) + } + + if isPerfectOverwrite { + glog.V(4).Infof("%s/%s overwrite2 [%d,%d)", pages.f.dir.Path, pages.f.Name, offset, offset+int64(len(data))) + pages.pages = []*DirtyPage{&DirtyPage{ + Offset: offset, + Data: data, + }} + } if pages.f.wfs.chunkSizeLimit > 0 && pages.totalSize() >= pages.f.wfs.chunkSizeLimit { chunk, err = pages.saveToStorage(ctx) pages.pages = nil - glog.V(3).Infof("%s/%s over size limit [%d,%d)", pages.f.dir.Path, pages.f.Name, chunk.Offset, chunk.Offset+int64(chunk.Size)) + glog.V(4).Infof("%s/%s over size limit [%d,%d)", pages.f.dir.Path, pages.f.Name, chunk.Offset, chunk.Offset+int64(chunk.Size)) } return } chunk, err = pages.saveToStorage(ctx) - glog.V(3).Infof("%s/%s saved [%d,%d)", pages.f.dir.Path, pages.f.Name, chunk.Offset, chunk.Offset+int64(chunk.Size)) + glog.V(4).Infof("%s/%s saved [%d,%d)", pages.f.dir.Path, pages.f.Name, chunk.Offset, chunk.Offset+int64(chunk.Size)) pages.pages = []*DirtyPage{&DirtyPage{ Offset: offset, @@ -87,7 +106,7 @@ func (pages *ContinuousDirtyPages) FlushToStorage(ctx context.Context) (chunk *f if chunk, err = pages.saveToStorage(ctx); err == nil { pages.pages = nil if chunk != nil { - glog.V(3).Infof("%s/%s flush [%d,%d)", pages.f.dir.Path, pages.f.Name, chunk.Offset, chunk.Offset+int64(chunk.Size)) + glog.V(4).Infof("%s/%s flush [%d,%d)", pages.f.dir.Path, pages.f.Name, chunk.Offset, chunk.Offset+int64(chunk.Size)) } } return diff --git a/weed/filesys/filehandle.go b/weed/filesys/filehandle.go index 52df2824c..9fa971236 100644 --- a/weed/filesys/filehandle.go +++ b/weed/filesys/filehandle.go @@ -39,7 +39,7 @@ var _ = fs.HandleReleaser(&FileHandle{}) func (fh *FileHandle) Read(ctx context.Context, req *fuse.ReadRequest, resp *fuse.ReadResponse) error { - glog.V(3).Infof("%v/%v read fh: [%d,%d)", fh.f.dir.Path, fh.f.Name, req.Offset, req.Offset+int64(req.Size)) + glog.V(4).Infof("%v/%v read fh: [%d,%d)", fh.f.dir.Path, fh.f.Name, req.Offset, req.Offset+int64(req.Size)) if len(fh.f.Chunks) == 0 { glog.V(0).Infof("empty fh %v/%v", fh.f.dir.Path, fh.f.Name) @@ -73,7 +73,7 @@ func (fh *FileHandle) Read(ctx context.Context, req *fuse.ReadRequest, resp *fus }) if err != nil { - glog.V(3).Infof("%v/%v read fh lookup volume ids: %v", fh.f.dir.Path, fh.f.Name, err) + glog.V(4).Infof("%v/%v read fh lookup volume ids: %v", fh.f.dir.Path, fh.f.Name, err) return fmt.Errorf("failed to lookup volume ids %v: %v", vids, err) } @@ -84,7 +84,7 @@ func (fh *FileHandle) Read(ctx context.Context, req *fuse.ReadRequest, resp *fus go func(chunkView *filer2.ChunkView) { defer wg.Done() - glog.V(3).Infof("read fh reading chunk: %+v", chunkView) + glog.V(4).Infof("read fh reading chunk: %+v", chunkView) locations := vid2Locations[volumeId(chunkView.FileId)] if locations == nil || len(locations.Locations) == 0 { @@ -109,7 +109,7 @@ func (fh *FileHandle) Read(ctx context.Context, req *fuse.ReadRequest, resp *fus return } - glog.V(3).Infof("read fh read %d bytes: %+v", n, chunkView) + glog.V(4).Infof("read fh read %d bytes: %+v", n, chunkView) totalRead += n }(chunkView) @@ -126,7 +126,7 @@ func (fh *FileHandle) Write(ctx context.Context, req *fuse.WriteRequest, resp *f // write the request to volume servers - glog.V(3).Infof("%+v/%v write fh: [%d,%d)", fh.f.dir.Path, fh.f.Name, req.Offset, req.Offset+int64(len(req.Data))) + glog.V(4).Infof("%+v/%v write fh: [%d,%d)", fh.f.dir.Path, fh.f.Name, req.Offset, req.Offset+int64(len(req.Data))) chunk, err := fh.dirtyPages.AddPage(ctx, req.Offset, req.Data) if err != nil { @@ -146,7 +146,7 @@ func (fh *FileHandle) Write(ctx context.Context, req *fuse.WriteRequest, resp *f func (fh *FileHandle) Release(ctx context.Context, req *fuse.ReleaseRequest) error { - glog.V(3).Infof("%+v/%v release fh", fh.f.dir.Path, fh.f.Name) + glog.V(4).Infof("%+v/%v release fh", fh.f.dir.Path, fh.f.Name) fh.f.isOpen = false @@ -158,7 +158,7 @@ func (fh *FileHandle) Release(ctx context.Context, req *fuse.ReleaseRequest) err func (fh *FileHandle) Flush(ctx context.Context, req *fuse.FlushRequest) error { // fflush works at fh level // send the data to the OS - glog.V(3).Infof("%s/%s fh flush %v", fh.f.dir.Path, fh.f.Name, req) + glog.V(4).Infof("%s/%s fh flush %v", fh.f.dir.Path, fh.f.Name, req) chunk, err := fh.dirtyPages.FlushToStorage(ctx) if err != nil { From 4e3ea49cffd2c65d1354c1406472027914775a49 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Tue, 29 May 2018 01:21:21 -0700 Subject: [PATCH 68/83] properly working local write buffer --- weed/command/mount.go | 2 +- weed/command/mount_std.go | 4 ++ weed/filesys/dir.go | 2 +- weed/filesys/dirty_page.go | 139 +++++++++++++------------------------ weed/filesys/file.go | 2 +- 5 files changed, 56 insertions(+), 93 deletions(-) diff --git a/weed/command/mount.go b/weed/command/mount.go index f0514b93a..6ba3b3697 100644 --- a/weed/command/mount.go +++ b/weed/command/mount.go @@ -18,7 +18,7 @@ func init() { mountOptions.dir = cmdMount.Flag.String("dir", ".", "mount weed filer to this directory") mountOptions.collection = cmdMount.Flag.String("collection", "", "collection to create the files") mountOptions.replication = cmdMount.Flag.String("replication", "000", "replication to create to files") - mountOptions.chunkSizeLimitMB = cmdMount.Flag.Int("chunkSizeLimitMB", 0, "if set, limit the chunk size in MB") + mountOptions.chunkSizeLimitMB = cmdMount.Flag.Int("chunkSizeLimitMB", 16, "local write buffer size, also chunk large files") } var cmdMount = &Command{ diff --git a/weed/command/mount_std.go b/weed/command/mount_std.go index 7e7a35f3d..d8b6884ff 100644 --- a/weed/command/mount_std.go +++ b/weed/command/mount_std.go @@ -19,6 +19,10 @@ func runMount(cmd *Command, args []string) bool { fmt.Printf("Please specify the mount directory via \"-dir\"") return false } + if *mountOptions.chunkSizeLimitMB <= 0 { + fmt.Printf("Please specify a reasonable buffer size.") + return false + } fuse.Unmount(*mountOptions.dir) diff --git a/weed/filesys/dir.go b/weed/filesys/dir.go index b90e428ab..bf4eda936 100644 --- a/weed/filesys/dir.go +++ b/weed/filesys/dir.go @@ -124,7 +124,7 @@ func (dir *Dir) Create(ctx context.Context, req *fuse.CreateRequest, file.isOpen = true return file, &FileHandle{ f: file, - dirtyPages: &ContinuousDirtyPages{f: file}, + dirtyPages: newDirtyPages(file), RequestId: req.Header.ID, NodeId: req.Header.Node, Uid: req.Uid, diff --git a/weed/filesys/dirty_page.go b/weed/filesys/dirty_page.go index 04e05b440..e3ee945a1 100644 --- a/weed/filesys/dirty_page.go +++ b/weed/filesys/dirty_page.go @@ -1,10 +1,8 @@ package filesys import ( - "sync" "fmt" "bytes" - "io" "time" "context" @@ -13,98 +11,64 @@ import ( "github.com/chrislusf/seaweedfs/weed/glog" ) -type DirtyPage struct { - Offset int64 - Data []byte -} - type ContinuousDirtyPages struct { - sync.Mutex - - pages []*DirtyPage - f *File + hasData bool + Offset int64 + Size int64 + Data []byte + f *File } -func (pages *ContinuousDirtyPages) AddPage(ctx context.Context, offset int64, data []byte) (chunk *filer_pb.FileChunk, err error) { - pages.Lock() - defer pages.Unlock() - - isPerfectOverwrite := false - isPerfectAppend := false - if len(pages.pages) > 0 { - lastPage := pages.pages[len(pages.pages)-1] - if lastPage.Offset+int64(len(lastPage.Data)) == offset { - // write continuous pages - glog.V(4).Infof("%s/%s append [%d,%d)", pages.f.dir.Path, pages.f.Name, offset, offset+int64(len(data))) - isPerfectAppend = true - } - if pages.pages[0].Offset == offset && pages.totalSize() == int64(len(data)) { - glog.V(4).Infof("%s/%s overwrite [%d,%d)", pages.f.dir.Path, pages.f.Name, offset, offset+int64(len(data))) - isPerfectOverwrite = true - } - } else { - glog.V(4).Infof("%s/%s append [%d,%d)", pages.f.dir.Path, pages.f.Name, offset, offset+int64(len(data))) - isPerfectAppend = true +func newDirtyPages(file *File) *ContinuousDirtyPages { + return &ContinuousDirtyPages{ + Data: make([]byte, file.wfs.chunkSizeLimit), + f: file, } +} - isPerfectReplace := false - for _, page := range pages.pages { - if page.Offset == offset && len(page.Data) == len(data) { - // perfect replace - glog.V(4).Infof("%s/%s replace [%d,%d)", pages.f.dir.Path, pages.f.Name, offset, offset+int64(len(data))) - page.Data = data - isPerfectReplace = true - } - } +func (pages *ContinuousDirtyPages) AddPage(ctx context.Context, offset int64, data []byte) (chunk *filer_pb.FileChunk, err error) { - if isPerfectReplace { - return nil, nil + if len(data) > len(pages.Data) { + // this is more than what we can hold. + panic("not prepared if buffer is smaller than each system write!") } - if isPerfectAppend || isPerfectOverwrite { - if isPerfectAppend { - glog.V(4).Infof("%s/%s append2 [%d,%d)", pages.f.dir.Path, pages.f.Name, offset, offset+int64(len(data))) - pages.pages = append(pages.pages, &DirtyPage{ - Offset: offset, - Data: data, - }) - } - - if isPerfectOverwrite { - glog.V(4).Infof("%s/%s overwrite2 [%d,%d)", pages.f.dir.Path, pages.f.Name, offset, offset+int64(len(data))) - pages.pages = []*DirtyPage{&DirtyPage{ - Offset: offset, - Data: data, - }} - } - - if pages.f.wfs.chunkSizeLimit > 0 && pages.totalSize() >= pages.f.wfs.chunkSizeLimit { - chunk, err = pages.saveToStorage(ctx) - pages.pages = nil - glog.V(4).Infof("%s/%s over size limit [%d,%d)", pages.f.dir.Path, pages.f.Name, chunk.Offset, chunk.Offset+int64(chunk.Size)) + if offset < pages.Offset || offset >= pages.Offset+int64(len(pages.Data)) || + pages.Offset+int64(len(pages.Data)) < offset+int64(len(data)) { + // if the data is out of range, + // or buffer is full if adding new data, + // flush current buffer and add new data + + // println("offset", offset, "size", len(data), "existing offset", pages.Offset, "size", pages.Size) + + if chunk, err = pages.saveToStorage(ctx); err == nil { + if chunk != nil { + glog.V(4).Infof("%s/%s add save [%d,%d)", pages.f.dir.Path, pages.f.Name, chunk.Offset, chunk.Offset+int64(chunk.Size)) + } + } else { + glog.V(0).Infof("%s/%s add save [%d,%d): %v", pages.f.dir.Path, pages.f.Name, chunk.Offset, chunk.Offset+int64(chunk.Size), err) + return } + pages.Offset = offset + pages.Size = int64(len(data)) + copy(pages.Data, data) return } - chunk, err = pages.saveToStorage(ctx) - - glog.V(4).Infof("%s/%s saved [%d,%d)", pages.f.dir.Path, pages.f.Name, chunk.Offset, chunk.Offset+int64(chunk.Size)) - - pages.pages = []*DirtyPage{&DirtyPage{ - Offset: offset, - Data: data, - }} + copy(pages.Data[offset-pages.Offset:], data) + pages.Size = max(pages.Size, offset+int64(len(data))-pages.Offset) return } func (pages *ContinuousDirtyPages) FlushToStorage(ctx context.Context) (chunk *filer_pb.FileChunk, err error) { - pages.Lock() - defer pages.Unlock() + if pages.Size == 0 { + return nil, nil + } if chunk, err = pages.saveToStorage(ctx); err == nil { - pages.pages = nil + pages.Size = 0 if chunk != nil { glog.V(4).Infof("%s/%s flush [%d,%d)", pages.f.dir.Path, pages.f.Name, chunk.Offset, chunk.Offset+int64(chunk.Size)) } @@ -112,16 +76,9 @@ func (pages *ContinuousDirtyPages) FlushToStorage(ctx context.Context) (chunk *f return } -func (pages *ContinuousDirtyPages) totalSize() (total int64) { - for _, page := range pages.pages { - total += int64(len(page.Data)) - } - return -} - func (pages *ContinuousDirtyPages) saveToStorage(ctx context.Context) (*filer_pb.FileChunk, error) { - if len(pages.pages) == 0 { + if pages.Size == 0 { return nil, nil } @@ -148,13 +105,8 @@ func (pages *ContinuousDirtyPages) saveToStorage(ctx context.Context) (*filer_pb return nil, fmt.Errorf("filer assign volume: %v", err) } - var readers []io.Reader - for _, page := range pages.pages { - readers = append(readers, bytes.NewReader(page.Data)) - } - fileUrl := fmt.Sprintf("http://%s/%s", host, fileId) - bufReader := io.MultiReader(readers...) + bufReader := bytes.NewReader(pages.Data[:pages.Size]) uploadResult, err := operation.Upload(fileUrl, pages.f.Name, bufReader, false, "application/octet-stream", nil, "") if err != nil { glog.V(0).Infof("upload data %v to %s: %v", pages.f.Name, fileUrl, err) @@ -167,9 +119,16 @@ func (pages *ContinuousDirtyPages) saveToStorage(ctx context.Context) (*filer_pb return &filer_pb.FileChunk{ FileId: fileId, - Offset: pages.pages[0].Offset, - Size: uint64(pages.totalSize()), + Offset: pages.Offset, + Size: uint64(pages.Size), Mtime: time.Now().UnixNano(), }, nil } + +func max(x, y int64) int64 { + if x > y { + return x + } + return y +} diff --git a/weed/filesys/file.go b/weed/filesys/file.go index 255fe4af0..1fb7d53b1 100644 --- a/weed/filesys/file.go +++ b/weed/filesys/file.go @@ -85,7 +85,7 @@ func (file *File) Open(ctx context.Context, req *fuse.OpenRequest, resp *fuse.Op return &FileHandle{ f: file, - dirtyPages: &ContinuousDirtyPages{f: file}, + dirtyPages: newDirtyPages(file), RequestId: req.Header.ID, NodeId: req.Header.Node, Uid: req.Uid, From 78aabf66cefe6dcbfd700b7098c81d5daef46969 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Tue, 29 May 2018 23:46:17 -0700 Subject: [PATCH 69/83] add error details --- weed/server/filer_server_handlers_admin.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/weed/server/filer_server_handlers_admin.go b/weed/server/filer_server_handlers_admin.go index 91a0e6fa0..0620e2f43 100644 --- a/weed/server/filer_server_handlers_admin.go +++ b/weed/server/filer_server_handlers_admin.go @@ -8,6 +8,7 @@ import ( "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "strconv" "time" + "fmt" ) func (fs *FilerServer) registerHandler(w http.ResponseWriter, r *http.Request) { @@ -16,19 +17,19 @@ func (fs *FilerServer) registerHandler(w http.ResponseWriter, r *http.Request) { fileSize, err := strconv.ParseUint(r.FormValue("fileSize"), 10, 64) if err != nil { glog.V(0).Infof("register %s to %s parse fileSize %s: %v", fileId, path, r.FormValue("fileSize"), err) - writeJsonError(w, r, http.StatusInternalServerError, err) + writeJsonError(w, r, http.StatusInternalServerError, fmt.Errorf("parsing fileSize: %v", err)) return } uid, err := strconv.ParseUint(r.FormValue("uid"), 10, 64) if err != nil && r.FormValue("uid") != "" { glog.V(0).Infof("register %s to %s parse uid %s: %v", fileId, path, r.FormValue("uid"), err) - writeJsonError(w, r, http.StatusInternalServerError, err) + writeJsonError(w, r, http.StatusInternalServerError, fmt.Errorf("parsing uid: %v", err)) return } gid, err := strconv.ParseUint(r.FormValue("gid"), 10, 64) if err != nil && r.FormValue("gid") != "" { glog.V(0).Infof("register %s to %s parse gid %s: %v", fileId, path, r.FormValue("gid"), err) - writeJsonError(w, r, http.StatusInternalServerError, err) + writeJsonError(w, r, http.StatusInternalServerError, fmt.Errorf("parsing gid: %v", err)) return } entry := &filer2.Entry{ @@ -50,7 +51,7 @@ func (fs *FilerServer) registerHandler(w http.ResponseWriter, r *http.Request) { err = fs.filer.CreateEntry(entry) if err != nil { glog.V(0).Infof("register %s to %s error: %v", fileId, path, err) - writeJsonError(w, r, http.StatusInternalServerError, err) + writeJsonError(w, r, http.StatusInternalServerError, fmt.Errorf("create %s: %v", path, err)) } else { w.WriteHeader(http.StatusOK) } From 8a48baa0563ac94b5e84b8b4fb2ffac34c8d8ff8 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Tue, 29 May 2018 23:46:45 -0700 Subject: [PATCH 70/83] add single chunk file copying to new filer --- weed/command/filer_copy.go | 74 +++++++++++++++++++++++++++++--------- 1 file changed, 57 insertions(+), 17 deletions(-) diff --git a/weed/command/filer_copy.go b/weed/command/filer_copy.go index 8a754bb55..3b8193cd1 100644 --- a/weed/command/filer_copy.go +++ b/weed/command/filer_copy.go @@ -11,6 +11,8 @@ import ( "github.com/chrislusf/seaweedfs/weed/operation" filer_operation "github.com/chrislusf/seaweedfs/weed/operation/filer" "github.com/chrislusf/seaweedfs/weed/security" + "path" + "net/http" ) var ( @@ -68,20 +70,20 @@ func runCopy(cmd *Command, args []string) bool { return false } filerDestination := args[len(args)-1] - fileOrDirs := args[0 : len(args)-1] + fileOrDirs := args[0: len(args)-1] filerUrl, err := url.Parse(filerDestination) if err != nil { fmt.Printf("The last argument should be a URL on filer: %v\n", err) return false } - path := filerUrl.Path - if !strings.HasSuffix(path, "/") { - path = path + "/" + urlPath := filerUrl.Path + if !strings.HasSuffix(urlPath, "/") { + urlPath = urlPath + "/" } for _, fileOrDir := range fileOrDirs { - if !doEachCopy(fileOrDir, filerUrl.Host, path) { + if !doEachCopy(fileOrDir, filerUrl.Host, urlPath) { return false } } @@ -120,29 +122,67 @@ func doEachCopy(fileOrDir string, host string, path string) bool { } } - parts, err := operation.NewFileParts([]string{fileOrDir}) + // find the chunk count + chunkSize := int64(*copy.maxMB * 1024 * 1024) + chunkCount := 1 + if chunkSize > 0 && fi.Size() > chunkSize { + chunkCount = int(fi.Size()/chunkSize) + 1 + } + + // assign a volume + assignResult, err := operation.Assign(*copy.master, &operation.VolumeAssignRequest{ + Count: uint64(chunkCount), + Replication: *copy.replication, + Collection: *copy.collection, + Ttl: *copy.ttl, + }) if err != nil { - fmt.Printf("Failed to read file %s: %v\n", fileOrDir, err) + fmt.Printf("Failed to assign from %s: %v\n", *copy.master, err) + } + + if chunkCount == 1 { + return uploadFileAsOne(host, path, assignResult, f, fi) } - results, err := operation.SubmitFiles(*copy.master, parts, - *copy.replication, *copy.collection, "", - *copy.ttl, *copy.maxMB, copy.secret) + return uploadFileInChunks(host, path, assignResult, f, chunkCount) +} + +func uploadFileAsOne(filerUrl string, urlPath string, assignResult *operation.AssignResult, f *os.File, fi os.FileInfo) bool { + // upload the file content + ext := strings.ToLower(path.Ext(f.Name())) + head := make([]byte, 512) + f.Seek(0, 0) + n, err := f.Read(head) if err != nil { - fmt.Printf("Failed to submit file %s: %v\n", fileOrDir, err) + fmt.Printf("read head of %v: %v\n", f.Name(), err) + return false } + f.Seek(0, 0) + mimeType := http.DetectContentType(head[:n]) + isGzipped := ext == ".gz" + + targetUrl := "http://" + assignResult.Url + "/" + assignResult.Fid - if strings.HasSuffix(path, "/") { - path = path + fi.Name() + uploadResult, err := operation.Upload(targetUrl, f.Name(), f, isGzipped, mimeType, nil, "") + if err != nil { + fmt.Printf("upload data %v to %s: %v\n", f.Name(), targetUrl, err) + return false + } + if uploadResult.Error != "" { + fmt.Printf("upload %v to %s result: %v\n", f.Name(), targetUrl, uploadResult.Error) + return false } - if err = filer_operation.RegisterFile(host, path, results[0].Fid, parts[0].FileSize, + if err = filer_operation.RegisterFile(filerUrl, filepath.Join(urlPath, f.Name()), assignResult.Fid, fi.Size(), os.Getuid(), os.Getgid(), copy.secret); err != nil { - fmt.Printf("Failed to register file %s on %s: %v\n", fileOrDir, host, err) + fmt.Printf("Failed to register file %s on %s: %v\n", f.Name(), filerUrl, err) return false } - fmt.Printf("Copy %s => http://%s%s\n", fileOrDir, host, path) - + fmt.Printf("Copied %s => http://%s%s\n", f.Name(), filerUrl, urlPath) return true } + +func uploadFileInChunks(filerUrl string, path string, assignResult *operation.AssignResult, f *os.File, chunkCount int) bool { + return false +} From 2d7dea1a091ef7d391188bae82c196e58980d535 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Tue, 29 May 2018 23:52:27 -0700 Subject: [PATCH 71/83] a little refactoring --- weed/command/filer_copy.go | 31 ++++++++++++++++++++----------- 1 file changed, 20 insertions(+), 11 deletions(-) diff --git a/weed/command/filer_copy.go b/weed/command/filer_copy.go index 3b8193cd1..5a878e675 100644 --- a/weed/command/filer_copy.go +++ b/weed/command/filer_copy.go @@ -149,17 +149,9 @@ func doEachCopy(fileOrDir string, host string, path string) bool { func uploadFileAsOne(filerUrl string, urlPath string, assignResult *operation.AssignResult, f *os.File, fi os.FileInfo) bool { // upload the file content - ext := strings.ToLower(path.Ext(f.Name())) - head := make([]byte, 512) - f.Seek(0, 0) - n, err := f.Read(head) - if err != nil { - fmt.Printf("read head of %v: %v\n", f.Name(), err) - return false - } - f.Seek(0, 0) - mimeType := http.DetectContentType(head[:n]) - isGzipped := ext == ".gz" + + mimeType := detectMimeType(f) + isGzipped := isGzipped(f.Name()) targetUrl := "http://" + assignResult.Url + "/" + assignResult.Fid @@ -186,3 +178,20 @@ func uploadFileAsOne(filerUrl string, urlPath string, assignResult *operation.As func uploadFileInChunks(filerUrl string, path string, assignResult *operation.AssignResult, f *os.File, chunkCount int) bool { return false } + +func isGzipped(filename string) bool { + return strings.ToLower(path.Ext(filename)) == ".gz" +} + +func detectMimeType(f *os.File) string { + head := make([]byte, 512) + f.Seek(0, 0) + n, err := f.Read(head) + if err != nil { + fmt.Printf("read head of %v: %v\n", f.Name(), err) + return "application/octet-stream" + } + f.Seek(0, 0) + mimeType := http.DetectContentType(head[:n]) + return mimeType +} From 2fe0d479f166f2a0b2605e5778605d8856c66811 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Wed, 30 May 2018 00:54:56 -0700 Subject: [PATCH 72/83] filer.copy supports chunking --- weed/command/filer_copy.go | 94 +++++++++++++++++++++++++++++++++++--- 1 file changed, 88 insertions(+), 6 deletions(-) diff --git a/weed/command/filer_copy.go b/weed/command/filer_copy.go index 5a878e675..86f43348f 100644 --- a/weed/command/filer_copy.go +++ b/weed/command/filer_copy.go @@ -13,6 +13,12 @@ import ( "github.com/chrislusf/seaweedfs/weed/security" "path" "net/http" + "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" + "strconv" + "io" + "time" + "google.golang.org/grpc" + "context" ) var ( @@ -144,10 +150,10 @@ func doEachCopy(fileOrDir string, host string, path string) bool { return uploadFileAsOne(host, path, assignResult, f, fi) } - return uploadFileInChunks(host, path, assignResult, f, chunkCount) + return uploadFileInChunks(host, path, assignResult, f, fi, chunkCount, chunkSize) } -func uploadFileAsOne(filerUrl string, urlPath string, assignResult *operation.AssignResult, f *os.File, fi os.FileInfo) bool { +func uploadFileAsOne(filerUrl string, urlFolder string, assignResult *operation.AssignResult, f *os.File, fi os.FileInfo) bool { // upload the file content mimeType := detectMimeType(f) @@ -164,19 +170,82 @@ func uploadFileAsOne(filerUrl string, urlPath string, assignResult *operation.As fmt.Printf("upload %v to %s result: %v\n", f.Name(), targetUrl, uploadResult.Error) return false } + fmt.Printf("uploaded %s to %s\n", f.Name(), targetUrl) - if err = filer_operation.RegisterFile(filerUrl, filepath.Join(urlPath, f.Name()), assignResult.Fid, fi.Size(), + if err = filer_operation.RegisterFile(filerUrl, filepath.Join(urlFolder, f.Name()), assignResult.Fid, fi.Size(), os.Getuid(), os.Getgid(), copy.secret); err != nil { fmt.Printf("Failed to register file %s on %s: %v\n", f.Name(), filerUrl, err) return false } - fmt.Printf("Copied %s => http://%s%s\n", f.Name(), filerUrl, urlPath) + fmt.Printf("copied %s => http://%s%s%s\n", f.Name(), filerUrl, urlFolder, f.Name()) return true } -func uploadFileInChunks(filerUrl string, path string, assignResult *operation.AssignResult, f *os.File, chunkCount int) bool { - return false +func uploadFileInChunks(filerUrl string, urlFolder string, assignResult *operation.AssignResult, f *os.File, fi os.FileInfo, chunkCount int, chunkSize int64) bool { + + var chunks []*filer_pb.FileChunk + + for i := int64(0); i < int64(chunkCount); i++ { + fileId := assignResult.Fid + if i > 0 { + fileId += "_" + strconv.FormatInt(i, 10) + } + + targetUrl := "http://" + assignResult.Url + "/" + fileId + + uploadResult, err := operation.Upload(targetUrl, + f.Name()+"-"+strconv.FormatInt(i+1, 10), + io.LimitReader(f, chunkSize), + false, "application/octet-stream", nil, "") + if err != nil { + fmt.Printf("upload data %v to %s: %v\n", f.Name(), targetUrl, err) + return false + } + if uploadResult.Error != "" { + fmt.Printf("upload %v to %s result: %v\n", f.Name(), targetUrl, uploadResult.Error) + return false + } + chunks = append(chunks, &filer_pb.FileChunk{ + FileId: fileId, + Offset: i * chunkSize, + Size: uint64(uploadResult.Size), + Mtime: time.Now().UnixNano(), + }) + fmt.Printf("uploaded %s split %d => %s\n", f.Name(), i, targetUrl) + } + + if err := withFilerClient(filerUrl, func(client filer_pb.SeaweedFilerClient) error { + request := &filer_pb.CreateEntryRequest{ + Directory: urlFolder, + Entry: &filer_pb.Entry{ + Name: f.Name(), + Attributes: &filer_pb.FuseAttributes{ + Crtime: time.Now().Unix(), + Mtime: time.Now().Unix(), + Gid: uint32(os.Getgid()), + Uid: uint32(os.Getuid()), + FileSize: uint64(fi.Size()), + FileMode: uint32(fi.Mode()), + }, + Chunks: chunks, + }, + } + + fmt.Printf("%s%s set chunks: %v", urlFolder, f.Name(), len(chunks)) + for i, chunk := range chunks { + fmt.Printf("%s%s chunks %d: %v [%d,%d)\n", urlFolder, f.Name(), i, chunk.FileId, chunk.Offset, chunk.Offset+int64(chunk.Size)) + } + if _, err := client.CreateEntry(context.Background(), request); err != nil { + return fmt.Errorf("update fh: %v", err) + } + return nil + }); err != nil { + fmt.Printf("upload data %v to http://%s%s%s: %v\n", f.Name(), filerUrl, urlFolder, f.Name(), err) + return false + } + + return true } func isGzipped(filename string) bool { @@ -195,3 +264,16 @@ func detectMimeType(f *os.File) string { mimeType := http.DetectContentType(head[:n]) return mimeType } + +func withFilerClient(filerAddress string, fn func(filer_pb.SeaweedFilerClient) error) error { + + grpcConnection, err := grpc.Dial(filerAddress, grpc.WithInsecure()) + if err != nil { + return fmt.Errorf("fail to dial %s: %v", filerAddress, err) + } + defer grpcConnection.Close() + + client := filer_pb.NewSeaweedFilerClient(grpcConnection) + + return fn(client) +} From 26e7cd8c7537dc010e6757060c1835a342fffc90 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Wed, 30 May 2018 01:05:26 -0700 Subject: [PATCH 73/83] assign a different volume on large file copying --- weed/command/filer_copy.go | 46 +++++++++++++++++++++----------------- 1 file changed, 26 insertions(+), 20 deletions(-) diff --git a/weed/command/filer_copy.go b/weed/command/filer_copy.go index 86f43348f..a61118bbf 100644 --- a/weed/command/filer_copy.go +++ b/weed/command/filer_copy.go @@ -135,9 +135,18 @@ func doEachCopy(fileOrDir string, host string, path string) bool { chunkCount = int(fi.Size()/chunkSize) + 1 } + if chunkCount == 1 { + return uploadFileAsOne(host, path, f, fi) + } + + return uploadFileInChunks(host, path, f, fi, chunkCount, chunkSize) +} + +func uploadFileAsOne(filerUrl string, urlFolder string, f *os.File, fi os.FileInfo) bool { + // assign a volume assignResult, err := operation.Assign(*copy.master, &operation.VolumeAssignRequest{ - Count: uint64(chunkCount), + Count: 1, Replication: *copy.replication, Collection: *copy.collection, Ttl: *copy.ttl, @@ -146,14 +155,6 @@ func doEachCopy(fileOrDir string, host string, path string) bool { fmt.Printf("Failed to assign from %s: %v\n", *copy.master, err) } - if chunkCount == 1 { - return uploadFileAsOne(host, path, assignResult, f, fi) - } - - return uploadFileInChunks(host, path, assignResult, f, fi, chunkCount, chunkSize) -} - -func uploadFileAsOne(filerUrl string, urlFolder string, assignResult *operation.AssignResult, f *os.File, fi os.FileInfo) bool { // upload the file content mimeType := detectMimeType(f) @@ -182,17 +183,24 @@ func uploadFileAsOne(filerUrl string, urlFolder string, assignResult *operation. return true } -func uploadFileInChunks(filerUrl string, urlFolder string, assignResult *operation.AssignResult, f *os.File, fi os.FileInfo, chunkCount int, chunkSize int64) bool { +func uploadFileInChunks(filerUrl string, urlFolder string, f *os.File, fi os.FileInfo, chunkCount int, chunkSize int64) bool { var chunks []*filer_pb.FileChunk for i := int64(0); i < int64(chunkCount); i++ { - fileId := assignResult.Fid - if i > 0 { - fileId += "_" + strconv.FormatInt(i, 10) + + // assign a volume + assignResult, err := operation.Assign(*copy.master, &operation.VolumeAssignRequest{ + Count: 1, + Replication: *copy.replication, + Collection: *copy.collection, + Ttl: *copy.ttl, + }) + if err != nil { + fmt.Printf("Failed to assign from %s: %v\n", *copy.master, err) } - targetUrl := "http://" + assignResult.Url + "/" + fileId + targetUrl := "http://" + assignResult.Url + "/" + assignResult.Fid uploadResult, err := operation.Upload(targetUrl, f.Name()+"-"+strconv.FormatInt(i+1, 10), @@ -207,12 +215,12 @@ func uploadFileInChunks(filerUrl string, urlFolder string, assignResult *operati return false } chunks = append(chunks, &filer_pb.FileChunk{ - FileId: fileId, + FileId: assignResult.Fid, Offset: i * chunkSize, Size: uint64(uploadResult.Size), Mtime: time.Now().UnixNano(), }) - fmt.Printf("uploaded %s split %d => %s\n", f.Name(), i, targetUrl) + fmt.Printf("uploaded %s-%d to %s [%d,%d)\n", f.Name(), i+1, targetUrl, i*chunkSize, i*chunkSize+int64(uploadResult.Size)) } if err := withFilerClient(filerUrl, func(client filer_pb.SeaweedFilerClient) error { @@ -232,10 +240,6 @@ func uploadFileInChunks(filerUrl string, urlFolder string, assignResult *operati }, } - fmt.Printf("%s%s set chunks: %v", urlFolder, f.Name(), len(chunks)) - for i, chunk := range chunks { - fmt.Printf("%s%s chunks %d: %v [%d,%d)\n", urlFolder, f.Name(), i, chunk.FileId, chunk.Offset, chunk.Offset+int64(chunk.Size)) - } if _, err := client.CreateEntry(context.Background(), request); err != nil { return fmt.Errorf("update fh: %v", err) } @@ -245,6 +249,8 @@ func uploadFileInChunks(filerUrl string, urlFolder string, assignResult *operati return false } + fmt.Printf("copied %s => http://%s%s%s\n", f.Name(), filerUrl, urlFolder, f.Name()) + return true } From 0301504184fa3342624c70b9066dfbfb7944af25 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Wed, 30 May 2018 20:24:57 -0700 Subject: [PATCH 74/83] add mime, use simple insert and update filer store API 1. add mime type to file in filer 2. purge old chunks if overwrite during insert --- weed/command/filer_copy.go | 5 +- .../filer2/abstract_sql/abstract_sql_store.go | 13 +- weed/filer2/cassandra/cassandra_store.go | 11 +- weed/filer2/entry.go | 1 + weed/filer2/entry_codec.go | 2 + weed/filer2/filer.go | 33 ++++- weed/filer2/filerstore.go | 2 +- weed/filer2/leveldb/leveldb_store.go | 11 +- weed/filer2/memdb/memdb_store.go | 10 +- weed/filer2/memdb/memdb_store_test.go | 2 +- weed/filer2/redis/redis_store.go | 13 +- weed/filesys/filehandle.go | 10 +- weed/operation/filer/register.go | 3 +- weed/pb/filer.proto | 1 + weed/pb/filer_pb/filer.pb.go | 124 ++++++++++-------- weed/server/filer_grpc_server.go | 13 +- weed/server/filer_server_handlers_admin.go | 2 + weed/server/filer_server_handlers_read.go | 8 +- weed/server/filer_server_handlers_write.go | 9 +- weed/server/filer_ui/templates.go | 8 +- 20 files changed, 149 insertions(+), 132 deletions(-) diff --git a/weed/command/filer_copy.go b/weed/command/filer_copy.go index a61118bbf..8d6b84ff2 100644 --- a/weed/command/filer_copy.go +++ b/weed/command/filer_copy.go @@ -174,7 +174,7 @@ func uploadFileAsOne(filerUrl string, urlFolder string, f *os.File, fi os.FileIn fmt.Printf("uploaded %s to %s\n", f.Name(), targetUrl) if err = filer_operation.RegisterFile(filerUrl, filepath.Join(urlFolder, f.Name()), assignResult.Fid, fi.Size(), - os.Getuid(), os.Getgid(), copy.secret); err != nil { + mimeType, os.Getuid(), os.Getgid(), copy.secret); err != nil { fmt.Printf("Failed to register file %s on %s: %v\n", f.Name(), filerUrl, err) return false } @@ -185,6 +185,8 @@ func uploadFileAsOne(filerUrl string, urlFolder string, f *os.File, fi os.FileIn func uploadFileInChunks(filerUrl string, urlFolder string, f *os.File, fi os.FileInfo, chunkCount int, chunkSize int64) bool { + mimeType := detectMimeType(f) + var chunks []*filer_pb.FileChunk for i := int64(0); i < int64(chunkCount); i++ { @@ -235,6 +237,7 @@ func uploadFileInChunks(filerUrl string, urlFolder string, f *os.File, fi os.Fil Uid: uint32(os.Getuid()), FileSize: uint64(fi.Size()), FileMode: uint32(fi.Mode()), + Mime: mimeType, }, Chunks: chunks, }, diff --git a/weed/filer2/abstract_sql/abstract_sql_store.go b/weed/filer2/abstract_sql/abstract_sql_store.go index aa3b82c5b..82ef571b6 100644 --- a/weed/filer2/abstract_sql/abstract_sql_store.go +++ b/weed/filer2/abstract_sql/abstract_sql_store.go @@ -77,26 +77,21 @@ func (store *AbstractSqlStore) FindEntry(fullpath filer2.FullPath) (*filer2.Entr return entry, nil } -func (store *AbstractSqlStore) DeleteEntry(fullpath filer2.FullPath) (*filer2.Entry, error) { - - entry, err := store.FindEntry(fullpath) - if err != nil { - return nil, nil - } +func (store *AbstractSqlStore) DeleteEntry(fullpath filer2.FullPath) (error) { dir, name := fullpath.DirAndName() res, err := store.DB.Exec(store.SqlDelete, hashToLong(dir), name, dir) if err != nil { - return nil, fmt.Errorf("delete %s: %s", fullpath, err) + return fmt.Errorf("delete %s: %s", fullpath, err) } _, err = res.RowsAffected() if err != nil { - return nil, fmt.Errorf("delete %s but no rows affected: %s", fullpath, err) + return fmt.Errorf("delete %s but no rows affected: %s", fullpath, err) } - return entry, nil + return nil } func (store *AbstractSqlStore) ListDirectoryEntries(fullpath filer2.FullPath, startFileName string, inclusive bool, limit int) (entries []*filer2.Entry, err error) { diff --git a/weed/filer2/cassandra/cassandra_store.go b/weed/filer2/cassandra/cassandra_store.go index a1e63b87d..ebbaedc1d 100644 --- a/weed/filer2/cassandra/cassandra_store.go +++ b/weed/filer2/cassandra/cassandra_store.go @@ -88,22 +88,17 @@ func (store *CassandraStore) FindEntry(fullpath filer2.FullPath) (entry *filer2. return entry, nil } -func (store *CassandraStore) DeleteEntry(fullpath filer2.FullPath) (entry *filer2.Entry, err error) { - - entry, err = store.FindEntry(fullpath) - if err != nil { - return nil, nil - } +func (store *CassandraStore) DeleteEntry(fullpath filer2.FullPath) error { dir, name := fullpath.DirAndName() if err := store.session.Query( "DELETE FROM filemeta WHERE directory=? AND name=?", dir, name).Exec(); err != nil { - return entry, fmt.Errorf("delete %s : %v", entry.FullPath, err) + return fmt.Errorf("delete %s : %v", fullpath, err) } - return entry, nil + return nil } func (store *CassandraStore) ListDirectoryEntries(fullpath filer2.FullPath, startFileName string, inclusive bool, diff --git a/weed/filer2/entry.go b/weed/filer2/entry.go index cf7ad08aa..a2f8b91ef 100644 --- a/weed/filer2/entry.go +++ b/weed/filer2/entry.go @@ -13,6 +13,7 @@ type Attr struct { Mode os.FileMode // file mode Uid uint32 // owner uid Gid uint32 // group gid + Mime string } func (attr Attr) IsDirectory() bool { diff --git a/weed/filer2/entry_codec.go b/weed/filer2/entry_codec.go index 8019b2193..7585203e9 100644 --- a/weed/filer2/entry_codec.go +++ b/weed/filer2/entry_codec.go @@ -17,6 +17,7 @@ func (entry *Entry) EncodeAttributesAndChunks() ([]byte, error) { FileMode: uint32(entry.Attr.Mode), Uid: entry.Uid, Gid: entry.Gid, + Mime: entry.Mime, }, Chunks: entry.Chunks, } @@ -36,6 +37,7 @@ func (entry *Entry) DecodeAttributesAndChunks(blob []byte) error { entry.Attr.Mode = os.FileMode(message.Attributes.FileMode) entry.Attr.Uid = message.Attributes.Uid entry.Attr.Gid = message.Attributes.Gid + entry.Attr.Mime = message.Attributes.Mime entry.Chunks = message.Chunks diff --git a/weed/filer2/filer.go b/weed/filer2/filer.go index f182adb13..0b4113c38 100644 --- a/weed/filer2/filer.go +++ b/weed/filer2/filer.go @@ -9,6 +9,7 @@ import ( "path/filepath" "strings" "time" + "github.com/chrislusf/seaweedfs/weed/operation" ) type Filer struct { @@ -101,6 +102,10 @@ func (f *Filer) CreateEntry(entry *Entry) error { } */ + if oldEntry, err := f.FindEntry(entry.FullPath); err == nil { + f.deleteChunks(oldEntry) + } + if err := f.store.InsertEntry(entry); err != nil { return fmt.Errorf("insert entry %s: %v", entry.FullPath, err) } @@ -116,26 +121,30 @@ func (f *Filer) FindEntry(p FullPath) (entry *Entry, err error) { return f.store.FindEntry(p) } -func (f *Filer) DeleteEntry(p FullPath) (fileEntry *Entry, err error) { +func (f *Filer) DeleteEntryMetaAndData(p FullPath) (err error) { entry, err := f.FindEntry(p) if err != nil { - return nil, err + return err } + if entry.IsDirectory() { entries, err := f.ListDirectoryEntries(p, "", false, 1) if err != nil { - return nil, fmt.Errorf("list folder %s: %v", p, err) + return fmt.Errorf("list folder %s: %v", p, err) } if len(entries) > 0 { - return nil, fmt.Errorf("folder %s is not empty", p) + return fmt.Errorf("folder %s is not empty", p) } } + + f.deleteChunks(entry) + return f.store.DeleteEntry(p) } func (f *Filer) ListDirectoryEntries(p FullPath, startFileName string, inclusive bool, limit int) ([]*Entry, error) { if strings.HasSuffix(string(p), "/") && len(p) > 1 { - p = p[0 : len(p)-1] + p = p[0: len(p)-1] } return f.store.ListDirectoryEntries(p, startFileName, inclusive, limit) } @@ -164,3 +173,17 @@ func (f *Filer) cacheSetDirectory(dirpath string, dirEntry *Entry, level int) { f.directoryCache.Set(dirpath, dirEntry, time.Duration(minutes)*time.Minute) } + +func (f *Filer) deleteChunks(entry *Entry) { + if f.master == "" { + return + } + if entry == nil { + return + } + for _, chunk := range entry.Chunks { + if err := operation.DeleteFile(f.master, chunk.FileId, ""); err != nil { + glog.V(0).Infof("deleting file %s: %v", chunk.FileId, err) + } + } +} diff --git a/weed/filer2/filerstore.go b/weed/filer2/filerstore.go index a5cd3352e..80822559b 100644 --- a/weed/filer2/filerstore.go +++ b/weed/filer2/filerstore.go @@ -11,7 +11,7 @@ type FilerStore interface { InsertEntry(*Entry) error UpdateEntry(*Entry) (err error) FindEntry(FullPath) (entry *Entry, err error) - DeleteEntry(FullPath) (fileEntry *Entry, err error) + DeleteEntry(FullPath) (err error) ListDirectoryEntries(dirPath FullPath, startFileName string, inclusive bool, limit int) ([]*Entry, error) } diff --git a/weed/filer2/leveldb/leveldb_store.go b/weed/filer2/leveldb/leveldb_store.go index a3125e923..58787714d 100644 --- a/weed/filer2/leveldb/leveldb_store.go +++ b/weed/filer2/leveldb/leveldb_store.go @@ -93,20 +93,15 @@ func (store *LevelDBStore) FindEntry(fullpath filer2.FullPath) (entry *filer2.En return entry, nil } -func (store *LevelDBStore) DeleteEntry(fullpath filer2.FullPath) (entry *filer2.Entry, err error) { +func (store *LevelDBStore) DeleteEntry(fullpath filer2.FullPath) (err error) { key := genKey(fullpath.DirAndName()) - entry, err = store.FindEntry(fullpath) - if err != nil { - return nil, nil - } - err = store.db.Delete(key, nil) if err != nil { - return entry, fmt.Errorf("delete %s : %v", entry.FullPath, err) + return fmt.Errorf("delete %s : %v", fullpath, err) } - return entry, nil + return nil } func (store *LevelDBStore) ListDirectoryEntries(fullpath filer2.FullPath, startFileName string, inclusive bool, diff --git a/weed/filer2/memdb/memdb_store.go b/weed/filer2/memdb/memdb_store.go index 098ed5dce..a8ef5cb39 100644 --- a/weed/filer2/memdb/memdb_store.go +++ b/weed/filer2/memdb/memdb_store.go @@ -56,13 +56,9 @@ func (store *MemDbStore) FindEntry(fullpath filer2.FullPath) (entry *filer2.Entr return entry, nil } -func (store *MemDbStore) DeleteEntry(fullpath filer2.FullPath) (entry *filer2.Entry, err error) { - item := store.tree.Delete(entryItem{&filer2.Entry{FullPath: fullpath}}) - if item == nil { - return nil, nil - } - entry = item.(entryItem).Entry - return entry, nil +func (store *MemDbStore) DeleteEntry(fullpath filer2.FullPath) (err error) { + store.tree.Delete(entryItem{&filer2.Entry{FullPath: fullpath}}) + return nil } func (store *MemDbStore) ListDirectoryEntries(fullpath filer2.FullPath, startFileName string, inclusive bool, limit int) (entries []*filer2.Entry, err error) { diff --git a/weed/filer2/memdb/memdb_store_test.go b/weed/filer2/memdb/memdb_store_test.go index bd1cdfdc0..5265ed248 100644 --- a/weed/filer2/memdb/memdb_store_test.go +++ b/weed/filer2/memdb/memdb_store_test.go @@ -134,7 +134,7 @@ func TestCreateFileAndList(t *testing.T) { } // delete file and count - filer.DeleteEntry(file3Path) + filer.DeleteEntryMetaAndData(file3Path) entries, _ = filer.ListDirectoryEntries(filer2.FullPath("/home/chris/this/is"), "", false, 100) if len(entries) != 1 { t.Errorf("list entries count: %v", len(entries)) diff --git a/weed/filer2/redis/redis_store.go b/weed/filer2/redis/redis_store.go index 5d1f51812..79a25096a 100644 --- a/weed/filer2/redis/redis_store.go +++ b/weed/filer2/redis/redis_store.go @@ -94,28 +94,23 @@ func (store *RedisStore) FindEntry(fullpath filer2.FullPath) (entry *filer2.Entr return entry, nil } -func (store *RedisStore) DeleteEntry(fullpath filer2.FullPath) (entry *filer2.Entry, err error) { - - entry, err = store.FindEntry(fullpath) - if err != nil { - return nil, nil - } +func (store *RedisStore) DeleteEntry(fullpath filer2.FullPath) (err error) { _, err = store.Client.Del(string(fullpath)).Result() if err != nil { - return entry, fmt.Errorf("delete %s : %v", entry.FullPath, err) + return fmt.Errorf("delete %s : %v", fullpath, err) } dir, name := fullpath.DirAndName() if name != "" { _, err = store.Client.SRem(genDirectoryListKey(dir), name).Result() if err != nil { - return nil, fmt.Errorf("delete %s in parent dir: %v", entry.FullPath, err) + return fmt.Errorf("delete %s in parent dir: %v", fullpath, err) } } - return entry, nil + return nil } func (store *RedisStore) ListDirectoryEntries(fullpath filer2.FullPath, startFileName string, inclusive bool, diff --git a/weed/filesys/filehandle.go b/weed/filesys/filehandle.go index 9fa971236..32e4622d0 100644 --- a/weed/filesys/filehandle.go +++ b/weed/filesys/filehandle.go @@ -11,6 +11,7 @@ import ( "github.com/chrislusf/seaweedfs/weed/util" "strings" "sync" + "net/http" ) type FileHandle struct { @@ -18,10 +19,6 @@ type FileHandle struct { dirtyPages *ContinuousDirtyPages dirtyMetadata bool - cachePath string - - handle uint64 - f *File RequestId fuse.RequestID // unique ID for request NodeId fuse.NodeID // file or directory the request is about @@ -135,6 +132,11 @@ func (fh *FileHandle) Write(ctx context.Context, req *fuse.WriteRequest, resp *f resp.Size = len(req.Data) + if req.Offset == 0 { + fh.f.attributes.Mime = http.DetectContentType(req.Data) + fh.dirtyMetadata = true + } + if chunk != nil { fh.f.Chunks = append(fh.f.Chunks, chunk) glog.V(1).Infof("uploaded %s/%s to %s [%d,%d)", fh.f.dir.Path, fh.f.Name, chunk.FileId, chunk.Offset, chunk.Offset+int64(chunk.Size)) diff --git a/weed/operation/filer/register.go b/weed/operation/filer/register.go index 108d93321..655fee1ff 100644 --- a/weed/operation/filer/register.go +++ b/weed/operation/filer/register.go @@ -17,7 +17,7 @@ type SubmitResult struct { Error string `json:"error,omitempty"` } -func RegisterFile(filer string, path string, fileId string, fileSize int64, uid, gid int, secret security.Secret) error { +func RegisterFile(filer string, path string, fileId string, fileSize int64, mime string, uid, gid int, secret security.Secret) error { // TODO: jwt need to be used _ = security.GenJwt(secret, fileId) @@ -27,6 +27,7 @@ func RegisterFile(filer string, path string, fileId string, fileSize int64, uid, values.Add("fileSize", strconv.FormatInt(fileSize, 10)) values.Add("uid", strconv.Itoa(uid)) values.Add("gid", strconv.Itoa(gid)) + values.Add("mime", mime) _, err := util.Post("http://"+filer+"/admin/register", values) if err != nil { return fmt.Errorf("Failed to register path %s on filer %s to file id %s : %v", path, filer, fileId, err) diff --git a/weed/pb/filer.proto b/weed/pb/filer.proto index 98c39d4b2..0f29ad02c 100644 --- a/weed/pb/filer.proto +++ b/weed/pb/filer.proto @@ -72,6 +72,7 @@ message FuseAttributes { uint32 uid = 4; uint32 gid = 5; int64 crtime = 6; + string mime = 7; } message GetEntryAttributesRequest { diff --git a/weed/pb/filer_pb/filer.pb.go b/weed/pb/filer_pb/filer.pb.go index 0473e77c0..74e43f519 100644 --- a/weed/pb/filer_pb/filer.pb.go +++ b/weed/pb/filer_pb/filer.pb.go @@ -214,6 +214,7 @@ type FuseAttributes struct { Uid uint32 `protobuf:"varint,4,opt,name=uid" json:"uid,omitempty"` Gid uint32 `protobuf:"varint,5,opt,name=gid" json:"gid,omitempty"` Crtime int64 `protobuf:"varint,6,opt,name=crtime" json:"crtime,omitempty"` + Mime string `protobuf:"bytes,7,opt,name=mime" json:"mime,omitempty"` } func (m *FuseAttributes) Reset() { *m = FuseAttributes{} } @@ -263,6 +264,13 @@ func (m *FuseAttributes) GetCrtime() int64 { return 0 } +func (m *FuseAttributes) GetMime() string { + if m != nil { + return m.Mime + } + return "" +} + type GetEntryAttributesRequest struct { Name string `protobuf:"bytes,1,opt,name=name" json:"name,omitempty"` ParentDir string `protobuf:"bytes,2,opt,name=parent_dir,json=parentDir" json:"parent_dir,omitempty"` @@ -931,62 +939,62 @@ var _SeaweedFiler_serviceDesc = grpc.ServiceDesc{ func init() { proto.RegisterFile("filer.proto", fileDescriptor0) } var fileDescriptor0 = []byte{ - // 899 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xb4, 0x56, 0xdd, 0x6e, 0xdc, 0x44, - 0x14, 0x8e, 0xd7, 0xf1, 0x26, 0x3e, 0xbb, 0xe1, 0x67, 0x36, 0x2d, 0x66, 0x9b, 0x54, 0x61, 0xa0, - 0xa8, 0x15, 0x52, 0x14, 0x05, 0x2e, 0x2a, 0x10, 0x12, 0x55, 0x53, 0xaa, 0x4a, 0xa9, 0x2a, 0xb9, - 0x04, 0x89, 0xab, 0x95, 0x63, 0x9f, 0x5d, 0x46, 0xf1, 0xda, 0xc6, 0x33, 0x0e, 0x0a, 0xb7, 0xbc, - 0x06, 0xb7, 0xbc, 0x01, 0x6f, 0xc0, 0x8b, 0xa1, 0xf9, 0xb1, 0x3d, 0x8e, 0xbd, 0xfd, 0xb9, 0xe0, - 0x6e, 0xe6, 0xfc, 0x7c, 0xe7, 0x3b, 0x33, 0x67, 0x3e, 0x1b, 0x26, 0x4b, 0x96, 0x62, 0x79, 0x5c, - 0x94, 0xb9, 0xc8, 0xc9, 0xae, 0xda, 0x2c, 0x8a, 0x4b, 0xfa, 0x0a, 0xee, 0x9d, 0xe7, 0xf9, 0x55, - 0x55, 0x9c, 0xb1, 0x12, 0x63, 0x91, 0x97, 0x37, 0xcf, 0x32, 0x51, 0xde, 0x84, 0xf8, 0x5b, 0x85, - 0x5c, 0x90, 0x03, 0xf0, 0x93, 0xda, 0x11, 0x38, 0x47, 0xce, 0x43, 0x3f, 0x6c, 0x0d, 0x84, 0xc0, - 0x76, 0x16, 0xad, 0x31, 0x18, 0x29, 0x87, 0x5a, 0xd3, 0x67, 0x70, 0x30, 0x0c, 0xc8, 0x8b, 0x3c, - 0xe3, 0x48, 0x1e, 0x80, 0x87, 0xd2, 0xa0, 0xd0, 0x26, 0xa7, 0x1f, 0x1e, 0xd7, 0x54, 0x8e, 0x75, - 0x9c, 0xf6, 0xd2, 0x53, 0x20, 0xe7, 0x8c, 0x0b, 0x69, 0x63, 0xc8, 0xdf, 0x89, 0x0e, 0xfd, 0x01, - 0x66, 0x9d, 0x1c, 0x53, 0xf1, 0x11, 0xec, 0xa0, 0x36, 0x05, 0xce, 0x91, 0x3b, 0x54, 0xb3, 0xf6, - 0xd3, 0xbf, 0x1d, 0xf0, 0x94, 0xa9, 0x69, 0xcd, 0x69, 0x5b, 0x23, 0x9f, 0xc1, 0x94, 0xf1, 0x45, - 0x4b, 0x40, 0xb6, 0xbd, 0x1b, 0x4e, 0x18, 0x6f, 0x5a, 0x25, 0x5f, 0xc1, 0x38, 0xfe, 0xb5, 0xca, - 0xae, 0x78, 0xe0, 0xaa, 0x52, 0xb3, 0xb6, 0xd4, 0x8f, 0x2c, 0xc5, 0xa7, 0xd2, 0x17, 0x9a, 0x10, - 0xf2, 0x18, 0x20, 0x12, 0xa2, 0x64, 0x97, 0x95, 0x40, 0x1e, 0x6c, 0xab, 0xf3, 0x08, 0xac, 0x84, - 0x8a, 0xe3, 0x93, 0xc6, 0x1f, 0x5a, 0xb1, 0x74, 0x09, 0x7e, 0x03, 0x47, 0x3e, 0x81, 0x1d, 0x99, - 0xb3, 0x60, 0x89, 0x61, 0x3b, 0x96, 0xdb, 0x17, 0x09, 0xb9, 0x0b, 0xe3, 0x7c, 0xb9, 0xe4, 0x28, - 0x14, 0x53, 0x37, 0x34, 0x3b, 0xd9, 0x1b, 0x67, 0x7f, 0x60, 0xe0, 0x1e, 0x39, 0x0f, 0xb7, 0x43, - 0xb5, 0x26, 0xfb, 0xe0, 0xad, 0x05, 0x5b, 0xa3, 0xa2, 0xe1, 0x86, 0x7a, 0x43, 0xff, 0x72, 0xe0, - 0x83, 0x2e, 0x0d, 0x72, 0x0f, 0x7c, 0x55, 0x4d, 0x21, 0x38, 0x0a, 0x41, 0x4d, 0xd3, 0xeb, 0x0e, - 0xca, 0xc8, 0x42, 0x69, 0x52, 0xd6, 0x79, 0xa2, 0x8b, 0xee, 0xe9, 0x94, 0x97, 0x79, 0x82, 0xe4, - 0x23, 0x70, 0x2b, 0x96, 0xa8, 0xb2, 0x7b, 0xa1, 0x5c, 0x4a, 0xcb, 0x8a, 0x25, 0x81, 0xa7, 0x2d, - 0x2b, 0xa6, 0x1a, 0x89, 0x4b, 0x85, 0x3b, 0xd6, 0x8d, 0xe8, 0x1d, 0x5d, 0xc1, 0xa7, 0xcf, 0x51, - 0xdd, 0xf7, 0x8d, 0x75, 0x50, 0x66, 0x56, 0x86, 0x6e, 0xf0, 0x10, 0xa0, 0x88, 0x4a, 0xcc, 0x84, - 0xbc, 0x45, 0x33, 0xb6, 0xbe, 0xb6, 0x9c, 0xb1, 0xd2, 0x3e, 0x49, 0xd7, 0x3e, 0x49, 0xfa, 0xa7, - 0x03, 0xf3, 0xa1, 0x4a, 0x66, 0xc2, 0xba, 0x17, 0xe9, 0xbc, 0xfb, 0x45, 0x5a, 0xf3, 0x32, 0x7a, - 0xeb, 0xbc, 0xd0, 0x13, 0xb8, 0xf3, 0x1c, 0x85, 0xb2, 0xe7, 0x99, 0xc0, 0x4c, 0xd4, 0xad, 0x6e, - 0x9a, 0x00, 0x7a, 0x0a, 0x77, 0x6f, 0x67, 0x18, 0xca, 0x01, 0xec, 0xc4, 0xda, 0xa4, 0x52, 0xa6, - 0x61, 0xbd, 0xa5, 0xbf, 0x00, 0x79, 0x5a, 0x62, 0x24, 0xf0, 0x3d, 0x84, 0xa0, 0x79, 0xd4, 0xa3, - 0x37, 0x3e, 0xea, 0x3b, 0x30, 0xeb, 0x40, 0x6b, 0x2e, 0xb2, 0xe2, 0x45, 0x91, 0xfc, 0x5f, 0x15, - 0x3b, 0xd0, 0xa6, 0x22, 0x03, 0x72, 0x86, 0x29, 0xbe, 0x57, 0xc5, 0x01, 0xb1, 0xeb, 0x29, 0x82, - 0xdb, 0x53, 0x04, 0xc9, 0xa0, 0x53, 0xca, 0x30, 0x58, 0xc3, 0xec, 0x09, 0xe7, 0x6c, 0x95, 0xfd, - 0x9c, 0xa7, 0xd5, 0x1a, 0x6b, 0x0a, 0xfb, 0xe0, 0xc5, 0x79, 0x65, 0x2e, 0xc5, 0x0b, 0xf5, 0x86, - 0xdc, 0x07, 0x88, 0xf3, 0x34, 0xc5, 0x58, 0xb0, 0x3c, 0x33, 0x04, 0x2c, 0x0b, 0x39, 0x82, 0x49, - 0x89, 0x45, 0xca, 0xe2, 0x48, 0x05, 0xe8, 0xd9, 0xb5, 0x4d, 0xf4, 0x1a, 0xf6, 0xbb, 0xe5, 0xcc, - 0x18, 0x6c, 0xd4, 0x0e, 0xf9, 0x2c, 0xcb, 0xd4, 0xd4, 0x92, 0x4b, 0xf5, 0x76, 0xaa, 0xcb, 0x94, - 0xc5, 0x0b, 0xe9, 0x70, 0xcd, 0xdb, 0x51, 0x96, 0x8b, 0x32, 0x6d, 0x99, 0x6f, 0x5b, 0xcc, 0xe9, - 0x37, 0x30, 0xd3, 0x5f, 0x83, 0x6e, 0x9b, 0x87, 0x00, 0xd7, 0xca, 0xb0, 0x60, 0x89, 0x56, 0x65, - 0x3f, 0xf4, 0xb5, 0xe5, 0x45, 0xc2, 0xe9, 0xf7, 0xe0, 0x9f, 0xe7, 0x9a, 0x39, 0x27, 0x27, 0xe0, - 0xa7, 0xf5, 0xc6, 0x08, 0x38, 0x69, 0x6f, 0xbb, 0x8e, 0x0b, 0xdb, 0x20, 0xfa, 0x1d, 0xec, 0xd6, - 0xe6, 0xba, 0x0f, 0x67, 0x53, 0x1f, 0xa3, 0x5b, 0x7d, 0xd0, 0x7f, 0x1d, 0xd8, 0xef, 0x52, 0x36, - 0x47, 0x75, 0x01, 0x7b, 0x4d, 0x89, 0xc5, 0x3a, 0x2a, 0x0c, 0x97, 0x13, 0x9b, 0x4b, 0x3f, 0xad, - 0x21, 0xc8, 0x5f, 0x46, 0x85, 0x1e, 0x81, 0x69, 0x6a, 0x99, 0xe6, 0x3f, 0xc1, 0xc7, 0xbd, 0x10, - 0xc9, 0xfa, 0x0a, 0xeb, 0x19, 0x94, 0x4b, 0xf2, 0x08, 0xbc, 0xeb, 0x28, 0xad, 0xd0, 0xcc, 0xfb, - 0xac, 0x7f, 0x02, 0x3c, 0xd4, 0x11, 0xdf, 0x8e, 0x1e, 0x3b, 0xa7, 0xff, 0x78, 0x30, 0x7d, 0x8d, - 0xd1, 0xef, 0x88, 0x89, 0x7c, 0xfd, 0x25, 0x59, 0xd5, 0x5d, 0x75, 0x3f, 0xcb, 0xe4, 0xc1, 0x6d, - 0xfa, 0x83, 0xff, 0x01, 0xf3, 0x2f, 0xdf, 0x16, 0x66, 0xc6, 0x7a, 0x8b, 0x9c, 0xc3, 0xc4, 0xfa, - 0x08, 0x93, 0x03, 0x2b, 0xb1, 0xf7, 0x3d, 0x9f, 0x1f, 0x6e, 0xf0, 0x36, 0x68, 0x11, 0x90, 0xbe, - 0xee, 0x92, 0xcf, 0xdb, 0xb4, 0x8d, 0xfa, 0x3f, 0xff, 0xe2, 0xcd, 0x41, 0x36, 0x61, 0x4b, 0x94, - 0x6c, 0xc2, 0x7d, 0x19, 0xb4, 0x09, 0x0f, 0x29, 0x99, 0x42, 0xb3, 0x04, 0xc7, 0x46, 0xeb, 0x4b, - 0x9c, 0x8d, 0x36, 0xa4, 0x52, 0x0a, 0xcd, 0x12, 0x0f, 0x1b, 0xad, 0x2f, 0x5f, 0x36, 0xda, 0x90, - 0xe2, 0x6c, 0x91, 0x57, 0x30, 0xb5, 0x45, 0x80, 0x58, 0x09, 0x03, 0x5a, 0x34, 0xbf, 0xbf, 0xc9, - 0x6d, 0x03, 0xda, 0x33, 0x6f, 0x03, 0x0e, 0xbc, 0x7a, 0x1b, 0x70, 0xe8, 0xa9, 0xd0, 0xad, 0xcb, - 0xb1, 0xfa, 0x3d, 0xfd, 0xfa, 0xbf, 0x00, 0x00, 0x00, 0xff, 0xff, 0x83, 0x92, 0xde, 0x64, 0xad, - 0x0a, 0x00, 0x00, + // 906 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0xb4, 0x56, 0x5b, 0x6f, 0xdc, 0x44, + 0x14, 0x8e, 0xd7, 0xd9, 0x4d, 0x7c, 0x76, 0xc3, 0x65, 0x36, 0x2d, 0x66, 0x9b, 0x54, 0x61, 0xa0, + 0xa8, 0x15, 0x52, 0x14, 0x05, 0x1e, 0x2a, 0x10, 0x12, 0x55, 0x53, 0xaa, 0x4a, 0xa9, 0x2a, 0xb9, + 0x04, 0x89, 0xa7, 0x95, 0x63, 0x9f, 0x5d, 0x46, 0xf1, 0x0d, 0xcf, 0x38, 0x28, 0xbc, 0xf2, 0x5b, + 0x78, 0xe7, 0x81, 0x7f, 0xc0, 0x1f, 0x43, 0x73, 0xb1, 0x3d, 0x8e, 0xbd, 0xbd, 0x3c, 0xf0, 0x36, + 0x73, 0x2e, 0xdf, 0xf9, 0xce, 0xcc, 0x99, 0xcf, 0x86, 0xe9, 0x8a, 0x25, 0x58, 0x1e, 0x17, 0x65, + 0x2e, 0x72, 0xb2, 0xab, 0x36, 0xcb, 0xe2, 0x92, 0xbe, 0x82, 0x7b, 0xe7, 0x79, 0x7e, 0x55, 0x15, + 0x67, 0xac, 0xc4, 0x48, 0xe4, 0xe5, 0xcd, 0xb3, 0x4c, 0x94, 0x37, 0x01, 0xfe, 0x56, 0x21, 0x17, + 0xe4, 0x00, 0xbc, 0xb8, 0x76, 0xf8, 0xce, 0x91, 0xf3, 0xd0, 0x0b, 0x5a, 0x03, 0x21, 0xb0, 0x9d, + 0x85, 0x29, 0xfa, 0x23, 0xe5, 0x50, 0x6b, 0xfa, 0x0c, 0x0e, 0x86, 0x01, 0x79, 0x91, 0x67, 0x1c, + 0xc9, 0x03, 0x18, 0xa3, 0x34, 0x28, 0xb4, 0xe9, 0xe9, 0x87, 0xc7, 0x35, 0x95, 0x63, 0x1d, 0xa7, + 0xbd, 0xf4, 0x14, 0xc8, 0x39, 0xe3, 0x42, 0xda, 0x18, 0xf2, 0x77, 0xa2, 0x43, 0x7f, 0x80, 0x79, + 0x27, 0xc7, 0x54, 0x7c, 0x04, 0x3b, 0xa8, 0x4d, 0xbe, 0x73, 0xe4, 0x0e, 0xd5, 0xac, 0xfd, 0xf4, + 0x2f, 0x07, 0xc6, 0xca, 0xd4, 0xb4, 0xe6, 0xb4, 0xad, 0x91, 0xcf, 0x60, 0xc6, 0xf8, 0xb2, 0x25, + 0x20, 0xdb, 0xde, 0x0d, 0xa6, 0x8c, 0x37, 0xad, 0x92, 0xaf, 0x60, 0x12, 0xfd, 0x5a, 0x65, 0x57, + 0xdc, 0x77, 0x55, 0xa9, 0x79, 0x5b, 0xea, 0x47, 0x96, 0xe0, 0x53, 0xe9, 0x0b, 0x4c, 0x08, 0x79, + 0x0c, 0x10, 0x0a, 0x51, 0xb2, 0xcb, 0x4a, 0x20, 0xf7, 0xb7, 0xd5, 0x79, 0xf8, 0x56, 0x42, 0xc5, + 0xf1, 0x49, 0xe3, 0x0f, 0xac, 0x58, 0xba, 0x02, 0xaf, 0x81, 0x23, 0x9f, 0xc0, 0x8e, 0xcc, 0x59, + 0xb2, 0xd8, 0xb0, 0x9d, 0xc8, 0xed, 0x8b, 0x98, 0xdc, 0x85, 0x49, 0xbe, 0x5a, 0x71, 0x14, 0x8a, + 0xa9, 0x1b, 0x98, 0x9d, 0xec, 0x8d, 0xb3, 0x3f, 0xd0, 0x77, 0x8f, 0x9c, 0x87, 0xdb, 0x81, 0x5a, + 0x93, 0x7d, 0x18, 0xa7, 0x82, 0xa5, 0xa8, 0x68, 0xb8, 0x81, 0xde, 0xd0, 0xbf, 0x1d, 0xf8, 0xa0, + 0x4b, 0x83, 0xdc, 0x03, 0x4f, 0x55, 0x53, 0x08, 0x8e, 0x42, 0x50, 0xd3, 0xf4, 0xba, 0x83, 0x32, + 0xb2, 0x50, 0x9a, 0x94, 0x34, 0x8f, 0x75, 0xd1, 0x3d, 0x9d, 0xf2, 0x32, 0x8f, 0x91, 0x7c, 0x04, + 0x6e, 0xc5, 0x62, 0x55, 0x76, 0x2f, 0x90, 0x4b, 0x69, 0x59, 0xb3, 0xd8, 0x1f, 0x6b, 0xcb, 0x9a, + 0xa9, 0x46, 0xa2, 0x52, 0xe1, 0x4e, 0x74, 0x23, 0x7a, 0x27, 0x1b, 0x49, 0xa5, 0x75, 0x47, 0x5f, + 0x92, 0x5c, 0xd3, 0x35, 0x7c, 0xfa, 0x1c, 0xd5, 0x0c, 0xdc, 0x58, 0x87, 0x67, 0xe6, 0x67, 0xe8, + 0x56, 0x0f, 0x01, 0x8a, 0xb0, 0xc4, 0x4c, 0xc8, 0x9b, 0x35, 0xa3, 0xec, 0x69, 0xcb, 0x19, 0x2b, + 0xed, 0xd3, 0x75, 0xed, 0xd3, 0xa5, 0x7f, 0x3a, 0xb0, 0x18, 0xaa, 0x64, 0xa6, 0xae, 0x7b, 0xb9, + 0xce, 0xbb, 0x5f, 0xae, 0x35, 0x43, 0xa3, 0xb7, 0xce, 0x10, 0x3d, 0x81, 0x3b, 0xcf, 0x51, 0x28, + 0x7b, 0x9e, 0x09, 0xcc, 0x44, 0xdd, 0xea, 0xa6, 0xa9, 0xa0, 0xa7, 0x70, 0xf7, 0x76, 0x86, 0xa1, + 0xec, 0xc3, 0x4e, 0xa4, 0x4d, 0x2a, 0x65, 0x16, 0xd4, 0x5b, 0xfa, 0x0b, 0x90, 0xa7, 0x25, 0x86, + 0x02, 0xdf, 0x43, 0x1c, 0x9a, 0x87, 0x3e, 0x7a, 0xe3, 0x43, 0xbf, 0x03, 0xf3, 0x0e, 0xb4, 0xe6, + 0x22, 0x2b, 0x5e, 0x14, 0xf1, 0xff, 0x55, 0xb1, 0x03, 0x6d, 0x2a, 0x32, 0x20, 0x67, 0x98, 0xe0, + 0x7b, 0x55, 0x1c, 0x10, 0xc0, 0x9e, 0x4a, 0xb8, 0x3d, 0x95, 0x90, 0x0c, 0x3a, 0xa5, 0x0c, 0x83, + 0x14, 0xe6, 0x4f, 0x38, 0x67, 0xeb, 0xec, 0xe7, 0x3c, 0xa9, 0x52, 0xac, 0x29, 0xec, 0xc3, 0x38, + 0xca, 0x2b, 0x73, 0x29, 0xe3, 0x40, 0x6f, 0xc8, 0x7d, 0x80, 0x28, 0x4f, 0x12, 0x8c, 0x04, 0xcb, + 0x33, 0x43, 0xc0, 0xb2, 0x90, 0x23, 0x98, 0x96, 0x58, 0x24, 0x2c, 0x0a, 0x55, 0x80, 0x9e, 0x5d, + 0xdb, 0x44, 0xaf, 0x61, 0xbf, 0x5b, 0xce, 0x8c, 0xc1, 0x46, 0x3d, 0x91, 0x4f, 0xb5, 0x4c, 0x4c, + 0x2d, 0xb9, 0x54, 0x6f, 0xa7, 0xba, 0x4c, 0x58, 0xb4, 0x94, 0x0e, 0xd7, 0xbc, 0x1d, 0x65, 0xb9, + 0x28, 0x93, 0x96, 0xf9, 0xb6, 0xc5, 0x9c, 0x7e, 0x03, 0x73, 0xfd, 0x85, 0xe8, 0xb6, 0x79, 0x08, + 0x70, 0xad, 0x0c, 0x4b, 0x16, 0x6b, 0xa5, 0xf6, 0x02, 0x4f, 0x5b, 0x5e, 0xc4, 0x9c, 0x7e, 0x0f, + 0xde, 0x79, 0xae, 0x99, 0x73, 0x72, 0x02, 0x5e, 0x52, 0x6f, 0x8c, 0xa8, 0x93, 0xf6, 0xb6, 0xeb, + 0xb8, 0xa0, 0x0d, 0xa2, 0xdf, 0xc1, 0x6e, 0x6d, 0xae, 0xfb, 0x70, 0x36, 0xf5, 0x31, 0xba, 0xd5, + 0x07, 0xfd, 0xd7, 0x81, 0xfd, 0x2e, 0x65, 0x73, 0x54, 0x17, 0xb0, 0xd7, 0x94, 0x58, 0xa6, 0x61, + 0x61, 0xb8, 0x9c, 0xd8, 0x5c, 0xfa, 0x69, 0x0d, 0x41, 0xfe, 0x32, 0x2c, 0xf4, 0x08, 0xcc, 0x12, + 0xcb, 0xb4, 0xf8, 0x09, 0x3e, 0xee, 0x85, 0x48, 0xd6, 0x57, 0x58, 0xcf, 0xa0, 0x5c, 0x92, 0x47, + 0x30, 0xbe, 0x0e, 0x93, 0x0a, 0xcd, 0xbc, 0xcf, 0xfb, 0x27, 0xc0, 0x03, 0x1d, 0xf1, 0xed, 0xe8, + 0xb1, 0x73, 0xfa, 0xcf, 0x18, 0x66, 0xaf, 0x31, 0xfc, 0x1d, 0x31, 0x96, 0xaf, 0xbf, 0x24, 0xeb, + 0xba, 0xab, 0xee, 0xa7, 0x9a, 0x3c, 0xb8, 0x4d, 0x7f, 0xf0, 0xdf, 0x60, 0xf1, 0xe5, 0xdb, 0xc2, + 0xcc, 0x58, 0x6f, 0x91, 0x73, 0x98, 0x5a, 0x1f, 0x66, 0x72, 0x60, 0x25, 0xf6, 0xbe, 0xf1, 0x8b, + 0xc3, 0x0d, 0xde, 0x06, 0x2d, 0x04, 0xd2, 0xd7, 0x5d, 0xf2, 0x79, 0x9b, 0xb6, 0x51, 0xff, 0x17, + 0x5f, 0xbc, 0x39, 0xc8, 0x26, 0x6c, 0x89, 0x92, 0x4d, 0xb8, 0x2f, 0x83, 0x36, 0xe1, 0x21, 0x25, + 0x53, 0x68, 0x96, 0xe0, 0xd8, 0x68, 0x7d, 0x89, 0xb3, 0xd1, 0x86, 0x54, 0x4a, 0xa1, 0x59, 0xe2, + 0x61, 0xa3, 0xf5, 0xe5, 0xcb, 0x46, 0x1b, 0x52, 0x9c, 0x2d, 0xf2, 0x0a, 0x66, 0xb6, 0x08, 0x10, + 0x2b, 0x61, 0x40, 0x8b, 0x16, 0xf7, 0x37, 0xb9, 0x6d, 0x40, 0x7b, 0xe6, 0x6d, 0xc0, 0x81, 0x57, + 0x6f, 0x03, 0x0e, 0x3d, 0x15, 0xba, 0x75, 0x39, 0x51, 0xbf, 0xac, 0x5f, 0xff, 0x17, 0x00, 0x00, + 0xff, 0xff, 0xf1, 0x42, 0x51, 0xbb, 0xc1, 0x0a, 0x00, 0x00, } diff --git a/weed/server/filer_grpc_server.go b/weed/server/filer_grpc_server.go index 44f5320cb..da0014af3 100644 --- a/weed/server/filer_grpc_server.go +++ b/weed/server/filer_grpc_server.go @@ -50,6 +50,7 @@ func (fs *FilerServer) ListEntries(ctx context.Context, req *filer_pb.ListEntrie Gid: entry.Gid, Uid: entry.Uid, FileMode: uint32(entry.Mode), + Mime: entry.Mime, }, }) } @@ -75,6 +76,7 @@ func (fs *FilerServer) GetEntryAttributes(ctx context.Context, req *filer_pb.Get attributes.Gid = entry.Gid attributes.Mtime = entry.Mtime.Unix() attributes.Crtime = entry.Crtime.Unix() + attributes.Mime = entry.Mime glog.V(3).Infof("GetEntryAttributes %v size %d chunks %d: %+v", fullpath, attributes.FileSize, len(entry.Chunks), attributes) @@ -120,6 +122,7 @@ func (fs *FilerServer) CreateEntry(ctx context.Context, req *filer_pb.CreateEntr Mode: os.FileMode(req.Entry.Attributes.FileMode), Uid: req.Entry.Attributes.Uid, Gid: req.Entry.Attributes.Gid, + Mime: req.Entry.Attributes.Mime, }, Chunks: req.Entry.Chunks, }) @@ -162,6 +165,7 @@ func (fs *FilerServer) UpdateEntry(ctx context.Context, req *filer_pb.UpdateEntr } newEntry.Attr.Uid = req.Entry.Attributes.Uid newEntry.Attr.Gid = req.Entry.Attributes.Gid + newEntry.Attr.Mime = req.Entry.Attributes.Mime } @@ -180,14 +184,7 @@ func (fs *FilerServer) UpdateEntry(ctx context.Context, req *filer_pb.UpdateEntr } func (fs *FilerServer) DeleteEntry(ctx context.Context, req *filer_pb.DeleteEntryRequest) (resp *filer_pb.DeleteEntryResponse, err error) { - entry, err := fs.filer.DeleteEntry(filer2.FullPath(filepath.Join(req.Directory, req.Name))) - if err == nil { - for _, chunk := range entry.Chunks { - if err = operation.DeleteFile(fs.getMasterNode(), chunk.FileId, fs.jwt(chunk.FileId)); err != nil { - glog.V(0).Infof("deleting file %s: %v", chunk.FileId, err) - } - } - } + err = fs.filer.DeleteEntryMetaAndData(filer2.FullPath(filepath.Join(req.Directory, req.Name))) return &filer_pb.DeleteEntryResponse{}, err } diff --git a/weed/server/filer_server_handlers_admin.go b/weed/server/filer_server_handlers_admin.go index 0620e2f43..93a2ab883 100644 --- a/weed/server/filer_server_handlers_admin.go +++ b/weed/server/filer_server_handlers_admin.go @@ -32,6 +32,7 @@ func (fs *FilerServer) registerHandler(w http.ResponseWriter, r *http.Request) { writeJsonError(w, r, http.StatusInternalServerError, fmt.Errorf("parsing gid: %v", err)) return } + mime := r.FormValue("mime") entry := &filer2.Entry{ FullPath: filer2.FullPath(path), Attr: filer2.Attr{ @@ -40,6 +41,7 @@ func (fs *FilerServer) registerHandler(w http.ResponseWriter, r *http.Request) { Mtime: time.Now(), Uid: uint32(uid), Gid: uint32(gid), + Mime: mime, }, Chunks: []*filer_pb.FileChunk{{ FileId: fileId, diff --git a/weed/server/filer_server_handlers_read.go b/weed/server/filer_server_handlers_read.go index 08430716c..7a208b560 100644 --- a/weed/server/filer_server_handlers_read.go +++ b/weed/server/filer_server_handlers_read.go @@ -111,9 +111,11 @@ func (fs *FilerServer) handleSingleChunk(w http.ResponseWriter, r *http.Request, func (fs *FilerServer) handleMultipleChunks(w http.ResponseWriter, r *http.Request, entry *filer2.Entry) { - mimeType := "" - if ext := path.Ext(entry.Name()); ext != "" { - mimeType = mime.TypeByExtension(ext) + mimeType := entry.Mime + if mimeType == "" { + if ext := path.Ext(entry.Name()); ext != "" { + mimeType = mime.TypeByExtension(ext) + } } if mimeType != "" { w.Header().Set("Content-Type", mimeType) diff --git a/weed/server/filer_server_handlers_write.go b/weed/server/filer_server_handlers_write.go index c4152ba4f..5d72b9808 100644 --- a/weed/server/filer_server_handlers_write.go +++ b/weed/server/filer_server_handlers_write.go @@ -195,19 +195,12 @@ func (fs *FilerServer) PostHandler(w http.ResponseWriter, r *http.Request) { // curl -X DELETE http://localhost:8888/path/to func (fs *FilerServer) DeleteHandler(w http.ResponseWriter, r *http.Request) { - entry, err := fs.filer.DeleteEntry(filer2.FullPath(r.URL.Path)) + err := fs.filer.DeleteEntryMetaAndData(filer2.FullPath(r.URL.Path)) if err != nil { glog.V(4).Infoln("deleting", r.URL.Path, ":", err.Error()) writeJsonError(w, r, http.StatusInternalServerError, err) return } - if entry != nil && !entry.IsDirectory() { - for _, chunk := range entry.Chunks { - oldFid := chunk.FileId - operation.DeleteFile(fs.getMasterNode(), oldFid, fs.jwt(oldFid)) - } - } - writeJsonQuiet(w, r, http.StatusAccepted, map[string]string{"error": ""}) } diff --git a/weed/server/filer_ui/templates.go b/weed/server/filer_ui/templates.go index d5c42672e..e5ef4b8b6 100644 --- a/weed/server/filer_ui/templates.go +++ b/weed/server/filer_ui/templates.go @@ -47,9 +47,15 @@ var StatusTpl = template.Must(template.New("status").Parse(` {{if $entry.IsDirectory}} + {{else}} + {{ $entry.Mime }} + {{end}} + + + {{if $entry.IsDirectory}} {{else}} {{ $entry.Size }} bytes -       +     {{end}} From 44acf4b756564100e8542f9db813fc868619cc87 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Wed, 30 May 2018 20:48:13 -0700 Subject: [PATCH 75/83] fix file name f.Name() is not the base file name --- weed/command/filer_copy.go | 33 +++++++++++++++++---------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/weed/command/filer_copy.go b/weed/command/filer_copy.go index 8d6b84ff2..c7747f9f0 100644 --- a/weed/command/filer_copy.go +++ b/weed/command/filer_copy.go @@ -156,35 +156,36 @@ func uploadFileAsOne(filerUrl string, urlFolder string, f *os.File, fi os.FileIn } // upload the file content - + fileName := filepath.Base(f.Name()) mimeType := detectMimeType(f) - isGzipped := isGzipped(f.Name()) + isGzipped := isGzipped(fileName) targetUrl := "http://" + assignResult.Url + "/" + assignResult.Fid - uploadResult, err := operation.Upload(targetUrl, f.Name(), f, isGzipped, mimeType, nil, "") + uploadResult, err := operation.Upload(targetUrl, fileName, f, isGzipped, mimeType, nil, "") if err != nil { - fmt.Printf("upload data %v to %s: %v\n", f.Name(), targetUrl, err) + fmt.Printf("upload data %v to %s: %v\n", fileName, targetUrl, err) return false } if uploadResult.Error != "" { - fmt.Printf("upload %v to %s result: %v\n", f.Name(), targetUrl, uploadResult.Error) + fmt.Printf("upload %v to %s result: %v\n", fileName, targetUrl, uploadResult.Error) return false } - fmt.Printf("uploaded %s to %s\n", f.Name(), targetUrl) + fmt.Printf("uploaded %s to %s\n", fileName, targetUrl) - if err = filer_operation.RegisterFile(filerUrl, filepath.Join(urlFolder, f.Name()), assignResult.Fid, fi.Size(), + if err = filer_operation.RegisterFile(filerUrl, filepath.Join(urlFolder, fileName), assignResult.Fid, fi.Size(), mimeType, os.Getuid(), os.Getgid(), copy.secret); err != nil { - fmt.Printf("Failed to register file %s on %s: %v\n", f.Name(), filerUrl, err) + fmt.Printf("Failed to register file %s on %s: %v\n", fileName, filerUrl, err) return false } - fmt.Printf("copied %s => http://%s%s%s\n", f.Name(), filerUrl, urlFolder, f.Name()) + fmt.Printf("copied %s => http://%s%s%s\n", fileName, filerUrl, urlFolder, fileName) return true } func uploadFileInChunks(filerUrl string, urlFolder string, f *os.File, fi os.FileInfo, chunkCount int, chunkSize int64) bool { + fileName := filepath.Base(f.Name()) mimeType := detectMimeType(f) var chunks []*filer_pb.FileChunk @@ -205,15 +206,15 @@ func uploadFileInChunks(filerUrl string, urlFolder string, f *os.File, fi os.Fil targetUrl := "http://" + assignResult.Url + "/" + assignResult.Fid uploadResult, err := operation.Upload(targetUrl, - f.Name()+"-"+strconv.FormatInt(i+1, 10), + fileName+"-"+strconv.FormatInt(i+1, 10), io.LimitReader(f, chunkSize), false, "application/octet-stream", nil, "") if err != nil { - fmt.Printf("upload data %v to %s: %v\n", f.Name(), targetUrl, err) + fmt.Printf("upload data %v to %s: %v\n", fileName, targetUrl, err) return false } if uploadResult.Error != "" { - fmt.Printf("upload %v to %s result: %v\n", f.Name(), targetUrl, uploadResult.Error) + fmt.Printf("upload %v to %s result: %v\n", fileName, targetUrl, uploadResult.Error) return false } chunks = append(chunks, &filer_pb.FileChunk{ @@ -222,14 +223,14 @@ func uploadFileInChunks(filerUrl string, urlFolder string, f *os.File, fi os.Fil Size: uint64(uploadResult.Size), Mtime: time.Now().UnixNano(), }) - fmt.Printf("uploaded %s-%d to %s [%d,%d)\n", f.Name(), i+1, targetUrl, i*chunkSize, i*chunkSize+int64(uploadResult.Size)) + fmt.Printf("uploaded %s-%d to %s [%d,%d)\n", fileName, i+1, targetUrl, i*chunkSize, i*chunkSize+int64(uploadResult.Size)) } if err := withFilerClient(filerUrl, func(client filer_pb.SeaweedFilerClient) error { request := &filer_pb.CreateEntryRequest{ Directory: urlFolder, Entry: &filer_pb.Entry{ - Name: f.Name(), + Name: fileName, Attributes: &filer_pb.FuseAttributes{ Crtime: time.Now().Unix(), Mtime: time.Now().Unix(), @@ -248,11 +249,11 @@ func uploadFileInChunks(filerUrl string, urlFolder string, f *os.File, fi os.Fil } return nil }); err != nil { - fmt.Printf("upload data %v to http://%s%s%s: %v\n", f.Name(), filerUrl, urlFolder, f.Name(), err) + fmt.Printf("upload data %v to http://%s%s%s: %v\n", fileName, filerUrl, urlFolder, fileName, err) return false } - fmt.Printf("copied %s => http://%s%s%s\n", f.Name(), filerUrl, urlFolder, f.Name()) + fmt.Printf("copied %s => http://%s%s%s\n", fileName, filerUrl, urlFolder, fileName) return true } From 430eb67489372eea51c39cda1414411a9c96df8c Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Wed, 30 May 2018 22:02:21 -0700 Subject: [PATCH 76/83] handle large file copy when write request is larger than buffer --- weed/filesys/dirty_page.go | 47 ++++++++++++++++++++++++++++++++------ weed/filesys/filehandle.go | 4 ++-- 2 files changed, 42 insertions(+), 9 deletions(-) diff --git a/weed/filesys/dirty_page.go b/weed/filesys/dirty_page.go index e3ee945a1..609af4181 100644 --- a/weed/filesys/dirty_page.go +++ b/weed/filesys/dirty_page.go @@ -26,11 +26,39 @@ func newDirtyPages(file *File) *ContinuousDirtyPages { } } -func (pages *ContinuousDirtyPages) AddPage(ctx context.Context, offset int64, data []byte) (chunk *filer_pb.FileChunk, err error) { +func (pages *ContinuousDirtyPages) AddPage(ctx context.Context, offset int64, data []byte) (chunks []*filer_pb.FileChunk, err error) { + + var chunk *filer_pb.FileChunk if len(data) > len(pages.Data) { // this is more than what we can hold. - panic("not prepared if buffer is smaller than each system write!") + + glog.V(0).Infof("not prepared if buffer is smaller than each system write! file %s [%d,%d)", pages.f.Name, offset, int64(len(data))+offset) + + // flush existing + if chunk, err = pages.saveExistingPagesToStorage(ctx); err == nil { + if chunk != nil { + glog.V(4).Infof("%s/%s flush existing [%d,%d)", pages.f.dir.Path, pages.f.Name, chunk.Offset, chunk.Offset+int64(chunk.Size)) + } + chunks = append(chunks, chunk) + } else { + glog.V(0).Infof("%s/%s failed to flush1 [%d,%d): %v", pages.f.dir.Path, pages.f.Name, chunk.Offset, chunk.Offset+int64(chunk.Size), err) + return + } + pages.Size = 0 + + // flush the big page + if chunk, err = pages.saveToStorage(ctx, data, offset); err == nil { + if chunk != nil { + glog.V(4).Infof("%s/%s flush big request [%d,%d)", pages.f.dir.Path, pages.f.Name, chunk.Offset, chunk.Offset+int64(chunk.Size)) + chunks = append(chunks, chunk) + } + } else { + glog.V(0).Infof("%s/%s failed to flush2 [%d,%d): %v", pages.f.dir.Path, pages.f.Name, chunk.Offset, chunk.Offset+int64(chunk.Size), err) + return + } + + return } if offset < pages.Offset || offset >= pages.Offset+int64(len(pages.Data)) || @@ -41,9 +69,10 @@ func (pages *ContinuousDirtyPages) AddPage(ctx context.Context, offset int64, da // println("offset", offset, "size", len(data), "existing offset", pages.Offset, "size", pages.Size) - if chunk, err = pages.saveToStorage(ctx); err == nil { + if chunk, err = pages.saveExistingPagesToStorage(ctx); err == nil { if chunk != nil { glog.V(4).Infof("%s/%s add save [%d,%d)", pages.f.dir.Path, pages.f.Name, chunk.Offset, chunk.Offset+int64(chunk.Size)) + chunks = append(chunks, chunk) } } else { glog.V(0).Infof("%s/%s add save [%d,%d): %v", pages.f.dir.Path, pages.f.Name, chunk.Offset, chunk.Offset+int64(chunk.Size), err) @@ -67,7 +96,7 @@ func (pages *ContinuousDirtyPages) FlushToStorage(ctx context.Context) (chunk *f return nil, nil } - if chunk, err = pages.saveToStorage(ctx); err == nil { + if chunk, err = pages.saveExistingPagesToStorage(ctx); err == nil { pages.Size = 0 if chunk != nil { glog.V(4).Infof("%s/%s flush [%d,%d)", pages.f.dir.Path, pages.f.Name, chunk.Offset, chunk.Offset+int64(chunk.Size)) @@ -76,7 +105,11 @@ func (pages *ContinuousDirtyPages) FlushToStorage(ctx context.Context) (chunk *f return } -func (pages *ContinuousDirtyPages) saveToStorage(ctx context.Context) (*filer_pb.FileChunk, error) { +func (pages *ContinuousDirtyPages) saveExistingPagesToStorage(ctx context.Context) (*filer_pb.FileChunk, error) { + return pages.saveToStorage(ctx, pages.Data[:pages.Size], pages.Offset) +} + +func (pages *ContinuousDirtyPages) saveToStorage(ctx context.Context, buf []byte, offset int64) (*filer_pb.FileChunk, error) { if pages.Size == 0 { return nil, nil @@ -119,8 +152,8 @@ func (pages *ContinuousDirtyPages) saveToStorage(ctx context.Context) (*filer_pb return &filer_pb.FileChunk{ FileId: fileId, - Offset: pages.Offset, - Size: uint64(pages.Size), + Offset: offset, + Size: uint64(len(buf)), Mtime: time.Now().UnixNano(), }, nil diff --git a/weed/filesys/filehandle.go b/weed/filesys/filehandle.go index 32e4622d0..bec240de2 100644 --- a/weed/filesys/filehandle.go +++ b/weed/filesys/filehandle.go @@ -125,7 +125,7 @@ func (fh *FileHandle) Write(ctx context.Context, req *fuse.WriteRequest, resp *f glog.V(4).Infof("%+v/%v write fh: [%d,%d)", fh.f.dir.Path, fh.f.Name, req.Offset, req.Offset+int64(len(req.Data))) - chunk, err := fh.dirtyPages.AddPage(ctx, req.Offset, req.Data) + chunks, err := fh.dirtyPages.AddPage(ctx, req.Offset, req.Data) if err != nil { return fmt.Errorf("write %s/%s at [%d,%d): %v", fh.f.dir.Path, fh.f.Name, req.Offset, req.Offset+int64(len(req.Data)), err) } @@ -137,7 +137,7 @@ func (fh *FileHandle) Write(ctx context.Context, req *fuse.WriteRequest, resp *f fh.dirtyMetadata = true } - if chunk != nil { + for _, chunk := range chunks { fh.f.Chunks = append(fh.f.Chunks, chunk) glog.V(1).Infof("uploaded %s/%s to %s [%d,%d)", fh.f.dir.Path, fh.f.Name, chunk.FileId, chunk.Offset, chunk.Offset+int64(chunk.Size)) fh.dirtyMetadata = true From a218eaf1f007f61b9769f712566504fe09702f34 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Wed, 30 May 2018 22:09:24 -0700 Subject: [PATCH 77/83] fix log --- weed/filesys/dirty_page.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/weed/filesys/dirty_page.go b/weed/filesys/dirty_page.go index 609af4181..996eb0abb 100644 --- a/weed/filesys/dirty_page.go +++ b/weed/filesys/dirty_page.go @@ -31,9 +31,7 @@ func (pages *ContinuousDirtyPages) AddPage(ctx context.Context, offset int64, da var chunk *filer_pb.FileChunk if len(data) > len(pages.Data) { - // this is more than what we can hold. - - glog.V(0).Infof("not prepared if buffer is smaller than each system write! file %s [%d,%d)", pages.f.Name, offset, int64(len(data))+offset) + // this is more than what buffer can hold. // flush existing if chunk, err = pages.saveExistingPagesToStorage(ctx); err == nil { From dc13e10637aa3c28509699dfe4a99465a1eaf976 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Wed, 30 May 2018 22:28:14 -0700 Subject: [PATCH 78/83] fix copy error on 0 size files --- weed/command/filer_copy.go | 88 +++++++++++++++++++++++++++----------- 1 file changed, 62 insertions(+), 26 deletions(-) diff --git a/weed/command/filer_copy.go b/weed/command/filer_copy.go index c7747f9f0..904aac76c 100644 --- a/weed/command/filer_copy.go +++ b/weed/command/filer_copy.go @@ -9,7 +9,6 @@ import ( "strings" "github.com/chrislusf/seaweedfs/weed/operation" - filer_operation "github.com/chrislusf/seaweedfs/weed/operation/filer" "github.com/chrislusf/seaweedfs/weed/security" "path" "net/http" @@ -144,42 +143,76 @@ func doEachCopy(fileOrDir string, host string, path string) bool { func uploadFileAsOne(filerUrl string, urlFolder string, f *os.File, fi os.FileInfo) bool { - // assign a volume - assignResult, err := operation.Assign(*copy.master, &operation.VolumeAssignRequest{ - Count: 1, - Replication: *copy.replication, - Collection: *copy.collection, - Ttl: *copy.ttl, - }) - if err != nil { - fmt.Printf("Failed to assign from %s: %v\n", *copy.master, err) - } - // upload the file content fileName := filepath.Base(f.Name()) mimeType := detectMimeType(f) isGzipped := isGzipped(fileName) - targetUrl := "http://" + assignResult.Url + "/" + assignResult.Fid + var chunks []*filer_pb.FileChunk - uploadResult, err := operation.Upload(targetUrl, fileName, f, isGzipped, mimeType, nil, "") - if err != nil { - fmt.Printf("upload data %v to %s: %v\n", fileName, targetUrl, err) - return false - } - if uploadResult.Error != "" { - fmt.Printf("upload %v to %s result: %v\n", fileName, targetUrl, uploadResult.Error) - return false + if fi.Size() > 0 { + + // assign a volume + assignResult, err := operation.Assign(*copy.master, &operation.VolumeAssignRequest{ + Count: 1, + Replication: *copy.replication, + Collection: *copy.collection, + Ttl: *copy.ttl, + }) + if err != nil { + fmt.Printf("Failed to assign from %s: %v\n", *copy.master, err) + } + + targetUrl := "http://" + assignResult.Url + "/" + assignResult.Fid + + uploadResult, err := operation.Upload(targetUrl, fileName, f, isGzipped, mimeType, nil, "") + if err != nil { + fmt.Printf("upload data %v to %s: %v\n", fileName, targetUrl, err) + return false + } + if uploadResult.Error != "" { + fmt.Printf("upload %v to %s result: %v\n", fileName, targetUrl, uploadResult.Error) + return false + } + fmt.Printf("uploaded %s to %s\n", fileName, targetUrl) + + chunks = append(chunks, &filer_pb.FileChunk{ + FileId: assignResult.Fid, + Offset: 0, + Size: uint64(uploadResult.Size), + Mtime: time.Now().UnixNano(), + }) + + fmt.Printf("copied %s => http://%s%s%s\n", fileName, filerUrl, urlFolder, fileName) } - fmt.Printf("uploaded %s to %s\n", fileName, targetUrl) - if err = filer_operation.RegisterFile(filerUrl, filepath.Join(urlFolder, fileName), assignResult.Fid, fi.Size(), - mimeType, os.Getuid(), os.Getgid(), copy.secret); err != nil { - fmt.Printf("Failed to register file %s on %s: %v\n", fileName, filerUrl, err) + if err := withFilerClient(filerUrl, func(client filer_pb.SeaweedFilerClient) error { + request := &filer_pb.CreateEntryRequest{ + Directory: urlFolder, + Entry: &filer_pb.Entry{ + Name: fileName, + Attributes: &filer_pb.FuseAttributes{ + Crtime: time.Now().Unix(), + Mtime: time.Now().Unix(), + Gid: uint32(os.Getgid()), + Uid: uint32(os.Getuid()), + FileSize: uint64(fi.Size()), + FileMode: uint32(fi.Mode()), + Mime: mimeType, + }, + Chunks: chunks, + }, + } + + if _, err := client.CreateEntry(context.Background(), request); err != nil { + return fmt.Errorf("update fh: %v", err) + } + return nil + }); err != nil { + fmt.Printf("upload data %v to http://%s%s%s: %v\n", fileName, filerUrl, urlFolder, fileName, err) return false } - fmt.Printf("copied %s => http://%s%s%s\n", fileName, filerUrl, urlFolder, fileName) return true } @@ -266,6 +299,9 @@ func detectMimeType(f *os.File) string { head := make([]byte, 512) f.Seek(0, 0) n, err := f.Read(head) + if err == io.EOF { + return "" + } if err != nil { fmt.Printf("read head of %v: %v\n", f.Name(), err) return "application/octet-stream" From 4d1eedfa28878d87bc318ada946480e8e2d67358 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Wed, 30 May 2018 22:29:18 -0700 Subject: [PATCH 79/83] prepare to merge with master --- weed/util/constants.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/weed/util/constants.go b/weed/util/constants.go index f96bfebca..2c1e5c830 100644 --- a/weed/util/constants.go +++ b/weed/util/constants.go @@ -1,5 +1,5 @@ package util const ( - VERSION = "0.78" + VERSION = "0.90 beta" ) From 7efeb146c53dc3baaac1e3b88c20f31d4d5f70fe Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Thu, 31 May 2018 22:49:55 -0700 Subject: [PATCH 80/83] fix log --- weed/server/filer_server_handlers_write.go | 2 +- weed/server/volume_server_handlers_admin.go | 4 ++-- weed/server/volume_server_handlers_vacuum.go | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/weed/server/filer_server_handlers_write.go b/weed/server/filer_server_handlers_write.go index 5d72b9808..7d93d4485 100644 --- a/weed/server/filer_server_handlers_write.go +++ b/weed/server/filer_server_handlers_write.go @@ -34,7 +34,7 @@ func (fs *FilerServer) queryFileInfoByPath(w http.ResponseWriter, r *http.Reques fileId = entry.Chunks[0].FileId urlLocation, err = operation.LookupFileId(fs.getMasterNode(), fileId) if err != nil { - glog.V(1).Infoln("operation LookupFileId %s failed, err is %s", fileId, err.Error()) + glog.V(1).Infof("operation LookupFileId %s failed, err is %s", fileId, err.Error()) w.WriteHeader(http.StatusNotFound) } } diff --git a/weed/server/volume_server_handlers_admin.go b/weed/server/volume_server_handlers_admin.go index f66ab2306..9a8b92ebc 100644 --- a/weed/server/volume_server_handlers_admin.go +++ b/weed/server/volume_server_handlers_admin.go @@ -25,7 +25,7 @@ func (vs *VolumeServer) assignVolumeHandler(w http.ResponseWriter, r *http.Reque if r.FormValue("preallocate") != "" { preallocate, err = strconv.ParseInt(r.FormValue("preallocate"), 10, 64) if err != nil { - glog.V(0).Infoln("ignoring invalid int64 value for preallocate = %v", r.FormValue("preallocate")) + glog.V(0).Infof("ignoring invalid int64 value for preallocate = %v", r.FormValue("preallocate")) } } err = vs.store.AddVolume( @@ -41,7 +41,7 @@ func (vs *VolumeServer) assignVolumeHandler(w http.ResponseWriter, r *http.Reque } else { writeJsonError(w, r, http.StatusNotAcceptable, err) } - glog.V(2).Infoln("assign volume = %s, collection = %s , replication = %s, error = %v", + glog.V(2).Infof("assign volume = %s, collection = %s , replication = %s, error = %v", r.FormValue("volume"), r.FormValue("collection"), r.FormValue("replication"), err) } diff --git a/weed/server/volume_server_handlers_vacuum.go b/weed/server/volume_server_handlers_vacuum.go index d50eba031..b45e97a50 100644 --- a/weed/server/volume_server_handlers_vacuum.go +++ b/weed/server/volume_server_handlers_vacuum.go @@ -22,7 +22,7 @@ func (vs *VolumeServer) vacuumVolumeCompactHandler(w http.ResponseWriter, r *htt if r.FormValue("preallocate") != "" { preallocate, err = strconv.ParseInt(r.FormValue("preallocate"), 10, 64) if err != nil { - glog.V(0).Infoln("Failed to parse int64 preallocate = %s: %v", r.FormValue("preallocate"), err) + glog.V(0).Infof("Failed to parse int64 preallocate = %s: %v", r.FormValue("preallocate"), err) } } err = vs.store.CompactVolume(r.FormValue("volume"), preallocate) From a6f7f9b0b8baedad7b1555c7e01a4ab3cec3784c Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Thu, 31 May 2018 22:50:19 -0700 Subject: [PATCH 81/83] add missing modify and create times --- weed/server/filer_server_handlers_write_autochunk.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/weed/server/filer_server_handlers_write_autochunk.go b/weed/server/filer_server_handlers_write_autochunk.go index a1f0fa27f..5156ae02c 100644 --- a/weed/server/filer_server_handlers_write_autochunk.go +++ b/weed/server/filer_server_handlers_write_autochunk.go @@ -158,7 +158,9 @@ func (fs *FilerServer) doAutoChunk(w http.ResponseWriter, r *http.Request, conte entry := &filer2.Entry{ FullPath: filer2.FullPath(path), Attr: filer2.Attr{ - Mode: 0660, + Mtime: time.Now(), + Crtime: time.Now(), + Mode: 0660, }, Chunks: fileChunks, } From 43e3f5724ca31918cd5a3d282639bd4f34e6bc58 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Fri, 1 Jun 2018 00:39:39 -0700 Subject: [PATCH 82/83] use fixed list of masters in both filer and volume servers --- weed/command/filer.go | 13 +- weed/command/server.go | 5 +- weed/command/volume.go | 9 +- weed/filer2/filer.go | 14 +- weed/filer2/filer_master.go | 60 ++++++++ weed/filer2/leveldb/leveldb_store_test.go | 2 +- weed/filer2/memdb/memdb_store_test.go | 4 +- weed/pb/master_pb/seaweed.pb.go | 143 ++++++++++++++---- weed/pb/seaweed.proto | 53 ++++--- weed/server/filer_grpc_server.go | 8 +- weed/server/filer_server.go | 94 ++---------- weed/server/filer_server_handlers_read.go | 4 +- weed/server/filer_server_handlers_write.go | 10 +- .../filer_server_handlers_write_autochunk.go | 2 +- weed/server/master_grpc_server.go | 12 ++ weed/server/volume_grpc_client.go | 52 ++++--- weed/server/volume_server.go | 33 ++-- weed/server/volume_server_handlers_read.go | 4 +- weed/server/volume_server_handlers_ui.go | 4 +- weed/server/volume_server_handlers_write.go | 6 +- weed/server/volume_server_ui/templates.go | 4 +- weed/storage/store.go | 51 ------- 22 files changed, 307 insertions(+), 280 deletions(-) create mode 100644 weed/filer2/filer_master.go diff --git a/weed/command/filer.go b/weed/command/filer.go index fd9419772..1bd1493bd 100644 --- a/weed/command/filer.go +++ b/weed/command/filer.go @@ -13,6 +13,7 @@ import ( "github.com/soheilhy/cmux" "google.golang.org/grpc" "google.golang.org/grpc/reflection" + "strings" ) var ( @@ -20,7 +21,7 @@ var ( ) type FilerOptions struct { - master *string + masters *string ip *string port *int publicPort *int @@ -34,7 +35,7 @@ type FilerOptions struct { func init() { cmdFiler.Run = runFiler // break init cycle - f.master = cmdFiler.Flag.String("master", "localhost:9333", "master server location") + f.masters = cmdFiler.Flag.String("master", "localhost:9333", "comma-separated master servers") f.collection = cmdFiler.Flag.String("collection", "", "all data will be stored in this collection") f.ip = cmdFiler.Flag.String("ip", "", "filer server http listen ip address") f.port = cmdFiler.Flag.Int("port", 8888, "filer server http listen port") @@ -47,8 +48,8 @@ func init() { } var cmdFiler = &Command{ - UsageLine: "filer -port=8888 -master=", - Short: "start a file server that points to a master server", + UsageLine: "filer -port=8888 -master=[,]*", + Short: "start a file server that points to a master server, or a list of master servers", Long: `start a file server which accepts REST operation for any files. //create or overwrite the file, the directories /path/to will be automatically created @@ -83,8 +84,10 @@ func (fo *FilerOptions) start() { publicVolumeMux = http.NewServeMux() } + masters := *f.masters + fs, nfs_err := weed_server.NewFilerServer(defaultMux, publicVolumeMux, - *fo.ip, *fo.port, *fo.master, *fo.collection, + *fo.ip, *fo.port, strings.Split(masters, ","), *fo.collection, *fo.defaultReplicaPlacement, *fo.redirectOnRead, *fo.disableDirListing, *fo.maxMB, *fo.secretKey, diff --git a/weed/command/server.go b/weed/command/server.go index 503b2c61d..606845199 100644 --- a/weed/command/server.go +++ b/weed/command/server.go @@ -83,7 +83,6 @@ var ( func init() { serverOptions.cpuprofile = cmdServer.Flag.String("cpuprofile", "", "cpu profile output file") - filerOptions.master = cmdServer.Flag.String("filer.master", "", "default to current master server") filerOptions.collection = cmdServer.Flag.String("filer.collection", "", "all data will be stored in this collection") filerOptions.port = cmdServer.Flag.Int("filer.port", 8888, "filer server http listen port") filerOptions.publicPort = cmdServer.Flag.Int("filer.port.public", 0, "filer server public http listen port") @@ -108,7 +107,7 @@ func runServer(cmd *Command, args []string) bool { *isStartingFiler = true } - *filerOptions.master = *serverIp + ":" + strconv.Itoa(*masterPort) + master := *serverIp + ":" + strconv.Itoa(*masterPort) filerOptions.ip = serverIp if *filerOptions.defaultReplicaPlacement == "" { @@ -251,7 +250,7 @@ func runServer(cmd *Command, args []string) bool { *serverIp, *volumePort, *volumeServerPublicUrl, folders, maxCounts, volumeNeedleMapKind, - *serverIp+":"+strconv.Itoa(*masterPort), *volumePulse, *serverDataCenter, *serverRack, + []string{master}, *volumePulse, *serverDataCenter, *serverRack, serverWhiteList, *volumeFixJpgOrientation, *volumeReadRedirect, ) diff --git a/weed/command/volume.go b/weed/command/volume.go index a54ffd1fd..407c39eb1 100644 --- a/weed/command/volume.go +++ b/weed/command/volume.go @@ -26,7 +26,7 @@ type VolumeServerOptions struct { ip *string publicUrl *string bindIp *string - master *string + masters *string pulseSeconds *int idleConnectionTimeout *int maxCpu *int @@ -47,7 +47,7 @@ func init() { v.ip = cmdVolume.Flag.String("ip", "", "ip or server name") v.publicUrl = cmdVolume.Flag.String("publicUrl", "", "Publicly accessible address") v.bindIp = cmdVolume.Flag.String("ip.bind", "0.0.0.0", "ip address to bind to") - v.master = cmdVolume.Flag.String("mserver", "localhost:9333", "master server location") + v.masters = cmdVolume.Flag.String("mserver", "localhost:9333", "comma-separated master servers") v.pulseSeconds = cmdVolume.Flag.Int("pulseSeconds", 5, "number of seconds between heartbeats, must be smaller than or equal to the master's setting") v.idleConnectionTimeout = cmdVolume.Flag.Int("idleTimeout", 30, "connection idle seconds") v.maxCpu = cmdVolume.Flag.Int("maxCpu", 0, "maximum number of CPUs. 0 means all available CPUs") @@ -132,11 +132,14 @@ func runVolume(cmd *Command, args []string) bool { case "btree": volumeNeedleMapKind = storage.NeedleMapBtree } + + masters := *v.masters + volumeServer := weed_server.NewVolumeServer(volumeMux, publicVolumeMux, *v.ip, *v.port, *v.publicUrl, v.folders, v.folderMaxLimits, volumeNeedleMapKind, - *v.master, *v.pulseSeconds, *v.dataCenter, *v.rack, + strings.Split(masters, ","), *v.pulseSeconds, *v.dataCenter, *v.rack, v.whiteList, *v.fixJpgOrientation, *v.readRedirect, ) diff --git a/weed/filer2/filer.go b/weed/filer2/filer.go index 0b4113c38..e886b7d74 100644 --- a/weed/filer2/filer.go +++ b/weed/filer2/filer.go @@ -13,14 +13,16 @@ import ( ) type Filer struct { - master string + masters []string store FilerStore directoryCache *ccache.Cache + + currentMaster string } -func NewFiler(master string) *Filer { +func NewFiler(masters []string) *Filer { return &Filer{ - master: master, + masters: masters, directoryCache: ccache.New(ccache.Configure().MaxSize(1000).ItemsToPrune(100)), } } @@ -175,14 +177,12 @@ func (f *Filer) cacheSetDirectory(dirpath string, dirEntry *Entry, level int) { } func (f *Filer) deleteChunks(entry *Entry) { - if f.master == "" { - return - } + if entry == nil { return } for _, chunk := range entry.Chunks { - if err := operation.DeleteFile(f.master, chunk.FileId, ""); err != nil { + if err := operation.DeleteFile(f.GetMaster(), chunk.FileId, ""); err != nil { glog.V(0).Infof("deleting file %s: %v", chunk.FileId, err) } } diff --git a/weed/filer2/filer_master.go b/weed/filer2/filer_master.go new file mode 100644 index 000000000..f69f68a85 --- /dev/null +++ b/weed/filer2/filer_master.go @@ -0,0 +1,60 @@ +package filer2 + +import ( + "fmt" + "context" + "time" + + "github.com/chrislusf/seaweedfs/weed/pb/master_pb" + "github.com/chrislusf/seaweedfs/weed/glog" + "google.golang.org/grpc" +) + +func (fs *Filer) GetMaster() string { + return fs.currentMaster +} + +func (fs *Filer) KeepConnectedToMaster() { + glog.V(0).Infof("Filer bootstraps with masters %v", fs.masters) + for _, master := range fs.masters { + glog.V(0).Infof("Connecting to %v", master) + withMasterClient(master, func(client master_pb.SeaweedClient) error { + stream, err := client.KeepConnected(context.Background()) + if err != nil { + glog.V(0).Infof("failed to keep connected to %s: %v", master, err) + return err + } + + glog.V(0).Infof("Connected to %v", master) + fs.currentMaster = master + + for { + time.Sleep(time.Duration(float32(10*1e3)*0.25) * time.Millisecond) + + if err = stream.Send(&master_pb.Empty{}); err != nil { + glog.V(0).Infof("failed to send to %s: %v", master, err) + return err + } + + if _, err = stream.Recv(); err != nil { + glog.V(0).Infof("failed to receive from %s: %v", master, err) + return err + } + } + }) + fs.currentMaster = "" + } +} + +func withMasterClient(master string, fn func(client master_pb.SeaweedClient) error) error { + + grpcConnection, err := grpc.Dial(master, grpc.WithInsecure()) + if err != nil { + return fmt.Errorf("fail to dial %s: %v", master, err) + } + defer grpcConnection.Close() + + client := master_pb.NewSeaweedClient(grpcConnection) + + return fn(client) +} diff --git a/weed/filer2/leveldb/leveldb_store_test.go b/weed/filer2/leveldb/leveldb_store_test.go index 896dabdc3..ad72a2e60 100644 --- a/weed/filer2/leveldb/leveldb_store_test.go +++ b/weed/filer2/leveldb/leveldb_store_test.go @@ -8,7 +8,7 @@ import ( ) func TestCreateAndFind(t *testing.T) { - filer := filer2.NewFiler("") + filer := filer2.NewFiler(nil) dir, _ := ioutil.TempDir("", "seaweedfs_filer_test") defer os.RemoveAll(dir) store := &LevelDBStore{} diff --git a/weed/filer2/memdb/memdb_store_test.go b/weed/filer2/memdb/memdb_store_test.go index 5265ed248..160b4a16d 100644 --- a/weed/filer2/memdb/memdb_store_test.go +++ b/weed/filer2/memdb/memdb_store_test.go @@ -6,7 +6,7 @@ import ( ) func TestCreateAndFind(t *testing.T) { - filer := filer2.NewFiler("") + filer := filer2.NewFiler(nil) store := &MemDbStore{} store.Initialize(nil) filer.SetStore(store) @@ -43,7 +43,7 @@ func TestCreateAndFind(t *testing.T) { } func TestCreateFileAndList(t *testing.T) { - filer := filer2.NewFiler("") + filer := filer2.NewFiler(nil) store := &MemDbStore{} store.Initialize(nil) filer.SetStore(store) diff --git a/weed/pb/master_pb/seaweed.pb.go b/weed/pb/master_pb/seaweed.pb.go index f02ffb8f8..a0f23225a 100644 --- a/weed/pb/master_pb/seaweed.pb.go +++ b/weed/pb/master_pb/seaweed.pb.go @@ -12,6 +12,7 @@ It has these top-level messages: Heartbeat HeartbeatResponse VolumeInformationMessage + Empty */ package master_pb @@ -235,10 +236,19 @@ func (m *VolumeInformationMessage) GetTtl() uint32 { return 0 } +type Empty struct { +} + +func (m *Empty) Reset() { *m = Empty{} } +func (m *Empty) String() string { return proto.CompactTextString(m) } +func (*Empty) ProtoMessage() {} +func (*Empty) Descriptor() ([]byte, []int) { return fileDescriptor0, []int{3} } + func init() { proto.RegisterType((*Heartbeat)(nil), "master_pb.Heartbeat") proto.RegisterType((*HeartbeatResponse)(nil), "master_pb.HeartbeatResponse") proto.RegisterType((*VolumeInformationMessage)(nil), "master_pb.VolumeInformationMessage") + proto.RegisterType((*Empty)(nil), "master_pb.Empty") } // Reference imports to suppress errors if they are not otherwise used. @@ -253,6 +263,7 @@ const _ = grpc.SupportPackageIsVersion4 type SeaweedClient interface { SendHeartbeat(ctx context.Context, opts ...grpc.CallOption) (Seaweed_SendHeartbeatClient, error) + KeepConnected(ctx context.Context, opts ...grpc.CallOption) (Seaweed_KeepConnectedClient, error) } type seaweedClient struct { @@ -294,10 +305,42 @@ func (x *seaweedSendHeartbeatClient) Recv() (*HeartbeatResponse, error) { return m, nil } +func (c *seaweedClient) KeepConnected(ctx context.Context, opts ...grpc.CallOption) (Seaweed_KeepConnectedClient, error) { + stream, err := grpc.NewClientStream(ctx, &_Seaweed_serviceDesc.Streams[1], c.cc, "/master_pb.Seaweed/KeepConnected", opts...) + if err != nil { + return nil, err + } + x := &seaweedKeepConnectedClient{stream} + return x, nil +} + +type Seaweed_KeepConnectedClient interface { + Send(*Empty) error + Recv() (*Empty, error) + grpc.ClientStream +} + +type seaweedKeepConnectedClient struct { + grpc.ClientStream +} + +func (x *seaweedKeepConnectedClient) Send(m *Empty) error { + return x.ClientStream.SendMsg(m) +} + +func (x *seaweedKeepConnectedClient) Recv() (*Empty, error) { + m := new(Empty) + if err := x.ClientStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + // Server API for Seaweed service type SeaweedServer interface { SendHeartbeat(Seaweed_SendHeartbeatServer) error + KeepConnected(Seaweed_KeepConnectedServer) error } func RegisterSeaweedServer(s *grpc.Server, srv SeaweedServer) { @@ -330,6 +373,32 @@ func (x *seaweedSendHeartbeatServer) Recv() (*Heartbeat, error) { return m, nil } +func _Seaweed_KeepConnected_Handler(srv interface{}, stream grpc.ServerStream) error { + return srv.(SeaweedServer).KeepConnected(&seaweedKeepConnectedServer{stream}) +} + +type Seaweed_KeepConnectedServer interface { + Send(*Empty) error + Recv() (*Empty, error) + grpc.ServerStream +} + +type seaweedKeepConnectedServer struct { + grpc.ServerStream +} + +func (x *seaweedKeepConnectedServer) Send(m *Empty) error { + return x.ServerStream.SendMsg(m) +} + +func (x *seaweedKeepConnectedServer) Recv() (*Empty, error) { + m := new(Empty) + if err := x.ServerStream.RecvMsg(m); err != nil { + return nil, err + } + return m, nil +} + var _Seaweed_serviceDesc = grpc.ServiceDesc{ ServiceName: "master_pb.Seaweed", HandlerType: (*SeaweedServer)(nil), @@ -341,6 +410,12 @@ var _Seaweed_serviceDesc = grpc.ServiceDesc{ ServerStreams: true, ClientStreams: true, }, + { + StreamName: "KeepConnected", + Handler: _Seaweed_KeepConnected_Handler, + ServerStreams: true, + ClientStreams: true, + }, }, Metadata: "seaweed.proto", } @@ -348,37 +423,39 @@ var _Seaweed_serviceDesc = grpc.ServiceDesc{ func init() { proto.RegisterFile("seaweed.proto", fileDescriptor0) } var fileDescriptor0 = []byte{ - // 504 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0x74, 0x93, 0xdf, 0x6e, 0xd3, 0x30, - 0x14, 0xc6, 0x49, 0x16, 0xda, 0xfa, 0x74, 0x1d, 0x9d, 0x85, 0x90, 0x05, 0x03, 0x4a, 0xb9, 0x89, - 0x04, 0xaa, 0xd0, 0xb8, 0xe6, 0x86, 0x49, 0x88, 0x69, 0x20, 0x26, 0x17, 0xb8, 0x8d, 0xdc, 0xe4, - 0x0c, 0x59, 0x73, 0xfe, 0xc8, 0x76, 0x47, 0xb3, 0x77, 0xe2, 0x2d, 0x78, 0x30, 0xe4, 0x93, 0xa6, - 0x9d, 0x10, 0xdc, 0x1d, 0xff, 0xce, 0xe7, 0xf8, 0xe4, 0xfb, 0x6c, 0x98, 0x38, 0x54, 0x3f, 0x11, - 0x8b, 0x45, 0x63, 0x6b, 0x5f, 0x73, 0x56, 0x2a, 0xe7, 0xd1, 0x66, 0xcd, 0x6a, 0xfe, 0x2b, 0x06, - 0xf6, 0x11, 0x95, 0xf5, 0x2b, 0x54, 0x9e, 0x1f, 0x41, 0xac, 0x1b, 0x11, 0xcd, 0xa2, 0x94, 0xc9, - 0x58, 0x37, 0x9c, 0x43, 0xd2, 0xd4, 0xd6, 0x8b, 0x78, 0x16, 0xa5, 0x13, 0x49, 0x35, 0x7f, 0x0a, - 0xd0, 0xac, 0x57, 0x46, 0xe7, 0xd9, 0xda, 0x1a, 0x71, 0x40, 0x5a, 0xd6, 0x91, 0x6f, 0xd6, 0xf0, - 0x14, 0xa6, 0xa5, 0xda, 0x64, 0x37, 0xb5, 0x59, 0x97, 0x98, 0xe5, 0xf5, 0xba, 0xf2, 0x22, 0xa1, - 0xed, 0x47, 0xa5, 0xda, 0x7c, 0x27, 0x7c, 0x16, 0x28, 0x9f, 0xc1, 0x61, 0x50, 0x5e, 0x69, 0x83, - 0xd9, 0x35, 0xb6, 0xe2, 0xfe, 0x2c, 0x4a, 0x13, 0x09, 0xa5, 0xda, 0x7c, 0xd0, 0x06, 0x2f, 0xb0, - 0xe5, 0xcf, 0x61, 0x5c, 0x28, 0xaf, 0xb2, 0x1c, 0x2b, 0x8f, 0x56, 0x0c, 0xe8, 0x2c, 0x08, 0xe8, - 0x8c, 0x48, 0x98, 0xcf, 0xaa, 0xfc, 0x5a, 0x0c, 0xa9, 0x43, 0x75, 0x98, 0x4f, 0x15, 0xa5, 0xae, - 0x32, 0x9a, 0x7c, 0x44, 0x47, 0x33, 0x22, 0x97, 0x61, 0xfc, 0x77, 0x30, 0xec, 0x66, 0x73, 0x82, - 0xcd, 0x0e, 0xd2, 0xf1, 0xe9, 0xcb, 0xc5, 0xce, 0x8d, 0x45, 0x37, 0xde, 0x79, 0x75, 0x55, 0xdb, - 0x52, 0x79, 0x5d, 0x57, 0x9f, 0xd1, 0x39, 0xf5, 0x03, 0x65, 0xbf, 0x67, 0xee, 0xe0, 0x78, 0x67, - 0x97, 0x44, 0xd7, 0xd4, 0x95, 0x43, 0x9e, 0xc2, 0x83, 0xae, 0xbf, 0xd4, 0xb7, 0xf8, 0x49, 0x97, - 0xda, 0x93, 0x87, 0x89, 0xfc, 0x1b, 0xf3, 0x13, 0x60, 0x0e, 0x73, 0x8b, 0xfe, 0x02, 0x5b, 0x72, - 0x95, 0xc9, 0x3d, 0xe0, 0x8f, 0x60, 0x60, 0x50, 0x15, 0x68, 0xb7, 0xb6, 0x6e, 0x57, 0xf3, 0xdf, - 0x31, 0x88, 0xff, 0x8d, 0x46, 0x99, 0x15, 0x74, 0xde, 0x44, 0xc6, 0xba, 0x08, 0x9e, 0x38, 0x7d, - 0x8b, 0xf4, 0xf5, 0x44, 0x52, 0xcd, 0x9f, 0x01, 0xe4, 0xb5, 0x31, 0x98, 0x87, 0x8d, 0xdb, 0x8f, - 0xdf, 0x21, 0xc1, 0x33, 0x8a, 0x61, 0x1f, 0x57, 0x22, 0x59, 0x20, 0x5d, 0x52, 0x2f, 0xe0, 0xb0, - 0x40, 0x83, 0xbe, 0x17, 0x74, 0x49, 0x8d, 0x3b, 0xd6, 0x49, 0x5e, 0x03, 0xef, 0x96, 0x45, 0xb6, - 0x6a, 0x77, 0xc2, 0x01, 0x09, 0xa7, 0xdb, 0xce, 0xfb, 0xb6, 0x57, 0x3f, 0x01, 0x66, 0x51, 0x15, - 0x59, 0x5d, 0x99, 0x96, 0xc2, 0x1b, 0xc9, 0x51, 0x00, 0x5f, 0x2a, 0xd3, 0xf2, 0x57, 0x70, 0x6c, - 0xb1, 0x31, 0x3a, 0x57, 0x59, 0x63, 0x54, 0x8e, 0x25, 0x56, 0x7d, 0x8e, 0xd3, 0x6d, 0xe3, 0xb2, - 0xe7, 0x5c, 0xc0, 0xf0, 0x06, 0xad, 0x0b, 0xbf, 0xc5, 0x48, 0xd2, 0x2f, 0xf9, 0x14, 0x0e, 0xbc, - 0x37, 0x02, 0x88, 0x86, 0xf2, 0xf4, 0x2b, 0x0c, 0x97, 0xdd, 0x3b, 0xe0, 0xe7, 0x30, 0x59, 0x62, - 0x55, 0xec, 0x6f, 0xfe, 0xc3, 0x3b, 0xb7, 0x60, 0x47, 0x1f, 0x9f, 0xfc, 0x8b, 0xf6, 0xb1, 0xcf, - 0xef, 0xa5, 0xd1, 0x9b, 0x68, 0x35, 0xa0, 0x37, 0xf5, 0xf6, 0x4f, 0x00, 0x00, 0x00, 0xff, 0xff, - 0x01, 0x14, 0xbb, 0x3a, 0x64, 0x03, 0x00, 0x00, + // 540 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x09, 0x6e, 0x88, 0x02, 0xff, 0x74, 0x93, 0xcf, 0x6e, 0xd3, 0x4e, + 0x10, 0xc7, 0x7f, 0x76, 0xdd, 0xa4, 0x9e, 0x34, 0xfd, 0xa5, 0x2b, 0x84, 0xac, 0x52, 0x20, 0x84, + 0x8b, 0x25, 0x50, 0x84, 0xca, 0x89, 0x03, 0x17, 0x22, 0x10, 0x55, 0x40, 0x54, 0x8e, 0xe0, 0x6a, + 0x6d, 0xec, 0x29, 0x5a, 0x75, 0xbd, 0xb6, 0x76, 0x37, 0x25, 0xee, 0x4b, 0xf0, 0x24, 0xbc, 0x05, + 0x0f, 0x86, 0x76, 0x36, 0x4e, 0x22, 0xfe, 0xdc, 0x66, 0x3f, 0xf3, 0x1d, 0xcf, 0x78, 0xbe, 0xbb, + 0x30, 0x34, 0xc8, 0xbf, 0x21, 0x96, 0xd3, 0x46, 0xd7, 0xb6, 0x66, 0x71, 0xc5, 0x8d, 0x45, 0x9d, + 0x37, 0xcb, 0xc9, 0x8f, 0x10, 0xe2, 0xf7, 0xc8, 0xb5, 0x5d, 0x22, 0xb7, 0xec, 0x04, 0x42, 0xd1, + 0x24, 0xc1, 0x38, 0x48, 0xe3, 0x2c, 0x14, 0x0d, 0x63, 0x10, 0x35, 0xb5, 0xb6, 0x49, 0x38, 0x0e, + 0xd2, 0x61, 0x46, 0x31, 0x7b, 0x08, 0xd0, 0xac, 0x96, 0x52, 0x14, 0xf9, 0x4a, 0xcb, 0xe4, 0x80, + 0xb4, 0xb1, 0x27, 0x9f, 0xb5, 0x64, 0x29, 0x8c, 0x2a, 0xbe, 0xce, 0x6f, 0x6b, 0xb9, 0xaa, 0x30, + 0x2f, 0xea, 0x95, 0xb2, 0x49, 0x44, 0xe5, 0x27, 0x15, 0x5f, 0x7f, 0x21, 0x3c, 0x73, 0x94, 0x8d, + 0xe1, 0xd8, 0x29, 0xaf, 0x85, 0xc4, 0xfc, 0x06, 0xdb, 0xe4, 0x70, 0x1c, 0xa4, 0x51, 0x06, 0x15, + 0x5f, 0xbf, 0x13, 0x12, 0xe7, 0xd8, 0xb2, 0xc7, 0x30, 0x28, 0xb9, 0xe5, 0x79, 0x81, 0xca, 0xa2, + 0x4e, 0x7a, 0xd4, 0x0b, 0x1c, 0x9a, 0x11, 0x71, 0xf3, 0x69, 0x5e, 0xdc, 0x24, 0x7d, 0xca, 0x50, + 0xec, 0xe6, 0xe3, 0x65, 0x25, 0x54, 0x4e, 0x93, 0x1f, 0x51, 0xeb, 0x98, 0xc8, 0x95, 0x1b, 0xff, + 0x35, 0xf4, 0xfd, 0x6c, 0x26, 0x89, 0xc7, 0x07, 0xe9, 0xe0, 0xe2, 0xe9, 0x74, 0xbb, 0x8d, 0xa9, + 0x1f, 0xef, 0x52, 0x5d, 0xd7, 0xba, 0xe2, 0x56, 0xd4, 0xea, 0x23, 0x1a, 0xc3, 0xbf, 0x62, 0xd6, + 0xd5, 0x4c, 0x0c, 0x9c, 0x6e, 0xd7, 0x95, 0xa1, 0x69, 0x6a, 0x65, 0x90, 0xa5, 0xf0, 0xbf, 0xcf, + 0x2f, 0xc4, 0x1d, 0x7e, 0x10, 0x95, 0xb0, 0xb4, 0xc3, 0x28, 0xfb, 0x1d, 0xb3, 0x73, 0x88, 0x0d, + 0x16, 0x1a, 0xed, 0x1c, 0x5b, 0xda, 0x6a, 0x9c, 0xed, 0x00, 0xbb, 0x0f, 0x3d, 0x89, 0xbc, 0x44, + 0xbd, 0x59, 0xeb, 0xe6, 0x34, 0xf9, 0x19, 0x42, 0xf2, 0xaf, 0xd1, 0xc8, 0xb3, 0x92, 0xfa, 0x0d, + 0xb3, 0x50, 0x94, 0x6e, 0x27, 0x46, 0xdc, 0x21, 0x7d, 0x3d, 0xca, 0x28, 0x66, 0x8f, 0x00, 0x8a, + 0x5a, 0x4a, 0x2c, 0x5c, 0xe1, 0xe6, 0xe3, 0x7b, 0xc4, 0xed, 0x8c, 0x6c, 0xd8, 0xd9, 0x15, 0x65, + 0xb1, 0x23, 0xde, 0xa9, 0x27, 0x70, 0x5c, 0xa2, 0x44, 0xdb, 0x09, 0xbc, 0x53, 0x03, 0xcf, 0xbc, + 0xe4, 0x39, 0x30, 0x7f, 0x2c, 0xf3, 0x65, 0xbb, 0x15, 0xf6, 0x48, 0x38, 0xda, 0x64, 0xde, 0xb4, + 0x9d, 0xfa, 0x01, 0xc4, 0x1a, 0x79, 0x99, 0xd7, 0x4a, 0xb6, 0x64, 0xde, 0x51, 0x76, 0xe4, 0xc0, + 0x27, 0x25, 0x5b, 0xf6, 0x0c, 0x4e, 0x35, 0x36, 0x52, 0x14, 0x3c, 0x6f, 0x24, 0x2f, 0xb0, 0x42, + 0xd5, 0xf9, 0x38, 0xda, 0x24, 0xae, 0x3a, 0xce, 0x12, 0xe8, 0xdf, 0xa2, 0x36, 0xee, 0xb7, 0x62, + 0x92, 0x74, 0x47, 0x36, 0x82, 0x03, 0x6b, 0x65, 0x02, 0x44, 0x5d, 0x38, 0xe9, 0xc3, 0xe1, 0xdb, + 0xaa, 0xb1, 0xed, 0xc5, 0xf7, 0x00, 0xfa, 0x0b, 0xff, 0x22, 0xd8, 0x25, 0x0c, 0x17, 0xa8, 0xca, + 0xdd, 0x1b, 0xb8, 0xb7, 0x77, 0x1f, 0xb6, 0xf4, 0xec, 0xfc, 0x6f, 0xb4, 0xbb, 0x00, 0x93, 0xff, + 0xd2, 0xe0, 0x45, 0xc0, 0x5e, 0xc1, 0x70, 0x8e, 0xd8, 0xcc, 0x6a, 0xa5, 0xb0, 0xb0, 0x58, 0xb2, + 0xd1, 0x5e, 0x11, 0x75, 0x3e, 0xfb, 0x83, 0xf8, 0xd2, 0x65, 0x8f, 0x1e, 0xe6, 0xcb, 0x5f, 0x01, + 0x00, 0x00, 0xff, 0xff, 0x41, 0x64, 0x8a, 0xd9, 0xa9, 0x03, 0x00, 0x00, } diff --git a/weed/pb/seaweed.proto b/weed/pb/seaweed.proto index dd0223f04..4d31f8e6a 100644 --- a/weed/pb/seaweed.proto +++ b/weed/pb/seaweed.proto @@ -5,37 +5,44 @@ package master_pb; ////////////////////////////////////////////////// service Seaweed { - rpc SendHeartbeat(stream Heartbeat) returns (stream HeartbeatResponse) {} + rpc SendHeartbeat (stream Heartbeat) returns (stream HeartbeatResponse) { + } + rpc KeepConnected (stream Empty) returns (stream Empty) { + } } ////////////////////////////////////////////////// message Heartbeat { - string ip = 1; - uint32 port = 2; - string public_url = 3; - uint32 max_volume_count = 4; - uint64 max_file_key = 5; - string data_center = 6; - string rack = 7; - uint32 admin_port = 8; - repeated VolumeInformationMessage volumes = 9; + string ip = 1; + uint32 port = 2; + string public_url = 3; + uint32 max_volume_count = 4; + uint64 max_file_key = 5; + string data_center = 6; + string rack = 7; + uint32 admin_port = 8; + repeated VolumeInformationMessage volumes = 9; } + message HeartbeatResponse { - uint64 volumeSizeLimit = 1; - string secretKey = 2; - string leader = 3; + uint64 volumeSizeLimit = 1; + string secretKey = 2; + string leader = 3; } message VolumeInformationMessage { - uint32 id = 1; - uint64 size = 2; - string collection = 3; - uint64 file_count = 4; - uint64 delete_count = 5; - uint64 deleted_byte_count = 6; - bool read_only = 7; - uint32 replica_placement = 8; - uint32 version = 9; - uint32 ttl = 10; + uint32 id = 1; + uint64 size = 2; + string collection = 3; + uint64 file_count = 4; + uint64 delete_count = 5; + uint64 deleted_byte_count = 6; + bool read_only = 7; + uint32 replica_placement = 8; + uint32 version = 9; + uint32 ttl = 10; +} + +message Empty { } diff --git a/weed/server/filer_grpc_server.go b/weed/server/filer_grpc_server.go index da0014af3..8a80cded5 100644 --- a/weed/server/filer_grpc_server.go +++ b/weed/server/filer_grpc_server.go @@ -88,7 +88,7 @@ func (fs *FilerServer) GetEntryAttributes(ctx context.Context, req *filer_pb.Get func (fs *FilerServer) LookupVolume(ctx context.Context, req *filer_pb.LookupVolumeRequest) (*filer_pb.LookupVolumeResponse, error) { - lookupResult, err := operation.LookupVolumeIds(fs.getMasterNode(), req.VolumeIds) + lookupResult, err := operation.LookupVolumeIds(fs.filer.GetMaster(), req.VolumeIds) if err != nil { return nil, err } @@ -172,11 +172,11 @@ func (fs *FilerServer) UpdateEntry(ctx context.Context, req *filer_pb.UpdateEntr if err = fs.filer.UpdateEntry(newEntry); err == nil { for _, garbage := range unusedChunks { glog.V(0).Infof("deleting %s old chunk: %v, [%d, %d)", fullpath, garbage.FileId, garbage.Offset, garbage.Offset+int64(garbage.Size)) - operation.DeleteFile(fs.master, garbage.FileId, fs.jwt(garbage.FileId)) + operation.DeleteFile(fs.filer.GetMaster(), garbage.FileId, fs.jwt(garbage.FileId)) } for _, garbage := range garbages { glog.V(0).Infof("deleting %s garbage chunk: %v, [%d, %d)", fullpath, garbage.FileId, garbage.Offset, garbage.Offset+int64(garbage.Size)) - operation.DeleteFile(fs.master, garbage.FileId, fs.jwt(garbage.FileId)) + operation.DeleteFile(fs.filer.GetMaster(), garbage.FileId, fs.jwt(garbage.FileId)) } } @@ -190,7 +190,7 @@ func (fs *FilerServer) DeleteEntry(ctx context.Context, req *filer_pb.DeleteEntr func (fs *FilerServer) AssignVolume(ctx context.Context, req *filer_pb.AssignVolumeRequest) (resp *filer_pb.AssignVolumeResponse, err error) { - assignResult, err := operation.Assign(fs.master, &operation.VolumeAssignRequest{ + assignResult, err := operation.Assign(fs.filer.GetMaster(), &operation.VolumeAssignRequest{ Count: uint64(req.Count), Replication: req.Replication, Collection: req.Collection, diff --git a/weed/server/filer_server.go b/weed/server/filer_server.go index 3e175e960..827971a0d 100644 --- a/weed/server/filer_server.go +++ b/weed/server/filer_server.go @@ -1,12 +1,8 @@ package weed_server import ( - "math/rand" "net/http" "strconv" - "sync" - "time" - "github.com/chrislusf/seaweedfs/weed/filer2" _ "github.com/chrislusf/seaweedfs/weed/filer2/cassandra" _ "github.com/chrislusf/seaweedfs/weed/filer2/leveldb" @@ -14,16 +10,13 @@ import ( _ "github.com/chrislusf/seaweedfs/weed/filer2/mysql" _ "github.com/chrislusf/seaweedfs/weed/filer2/postgres" _ "github.com/chrislusf/seaweedfs/weed/filer2/redis" - "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/security" - "github.com/chrislusf/seaweedfs/weed/storage" - "github.com/chrislusf/seaweedfs/weed/util" + "github.com/chrislusf/seaweedfs/weed/glog" ) type FilerServer struct { port string - master string - mnLock sync.RWMutex + masters []string collection string defaultReplication string redirectOnRead bool @@ -31,16 +24,15 @@ type FilerServer struct { secret security.Secret filer *filer2.Filer maxMB int - masterNodes *storage.MasterNodes } -func NewFilerServer(defaultMux, readonlyMux *http.ServeMux, ip string, port int, master string, collection string, +func NewFilerServer(defaultMux, readonlyMux *http.ServeMux, ip string, port int, masters []string, collection string, replication string, redirectOnRead bool, disableDirListing bool, maxMB int, secret string, ) (fs *FilerServer, err error) { fs = &FilerServer{ - master: master, + masters: masters, collection: collection, defaultReplication: replication, redirectOnRead: redirectOnRead, @@ -48,7 +40,14 @@ func NewFilerServer(defaultMux, readonlyMux *http.ServeMux, ip string, port int, maxMB: maxMB, port: ip + ":" + strconv.Itoa(port), } - fs.filer = filer2.NewFiler(master) + + if len(masters) == 0 { + glog.Fatal("master list is required!") + } + + fs.filer = filer2.NewFiler(masters) + + go fs.filer.KeepConnectedToMaster() fs.filer.LoadConfiguration() @@ -59,78 +58,9 @@ func NewFilerServer(defaultMux, readonlyMux *http.ServeMux, ip string, port int, readonlyMux.HandleFunc("/", fs.readonlyFilerHandler) } - go func() { - connected := true - - fs.masterNodes = storage.NewMasterNodes(fs.master) - glog.V(0).Infof("Filer server bootstraps with master %s", fs.getMasterNode()) - - for { - glog.V(4).Infof("Filer server sending to master %s", fs.getMasterNode()) - master, err := fs.detectHealthyMaster(fs.getMasterNode()) - if err == nil { - if !connected { - connected = true - if fs.getMasterNode() != master { - fs.setMasterNode(master) - } - glog.V(0).Infoln("Filer Server Connected with master at", master) - } - } else { - glog.V(1).Infof("Filer Server Failed to talk with master %s: %v", fs.getMasterNode(), err) - if connected { - connected = false - } - } - if connected { - time.Sleep(time.Duration(float32(10*1e3)*(1+rand.Float32())) * time.Millisecond) - } else { - time.Sleep(time.Duration(float32(10*1e3)*0.25) * time.Millisecond) - } - } - }() - return fs, nil } func (fs *FilerServer) jwt(fileId string) security.EncodedJwt { return security.GenJwt(fs.secret, fileId) } - -func (fs *FilerServer) getMasterNode() string { - fs.mnLock.RLock() - defer fs.mnLock.RUnlock() - return fs.master -} - -func (fs *FilerServer) setMasterNode(masterNode string) { - fs.mnLock.Lock() - defer fs.mnLock.Unlock() - fs.master = masterNode -} - -func (fs *FilerServer) detectHealthyMaster(masterNode string) (master string, e error) { - if e = checkMaster(masterNode); e != nil { - fs.masterNodes.Reset() - for i := 0; i <= 3; i++ { - master, e = fs.masterNodes.FindMaster() - if e != nil { - continue - } else { - if e = checkMaster(master); e == nil { - break - } - } - } - } else { - master = masterNode - } - return -} - -func checkMaster(masterNode string) error { - statUrl := "http://" + masterNode + "/stats/health" - glog.V(4).Infof("Connecting to %s ...", statUrl) - _, e := util.Get(statUrl) - return e -} diff --git a/weed/server/filer_server_handlers_read.go b/weed/server/filer_server_handlers_read.go index 7a208b560..c690575b6 100644 --- a/weed/server/filer_server_handlers_read.go +++ b/weed/server/filer_server_handlers_read.go @@ -63,7 +63,7 @@ func (fs *FilerServer) handleSingleChunk(w http.ResponseWriter, r *http.Request, fileId := entry.Chunks[0].FileId - urlString, err := operation.LookupFileId(fs.getMasterNode(), fileId) + urlString, err := operation.LookupFileId(fs.filer.GetMaster(), fileId) if err != nil { glog.V(1).Infof("operation LookupFileId %s failed, err: %v", fileId, err) w.WriteHeader(http.StatusNotFound) @@ -225,7 +225,7 @@ func (fs *FilerServer) writeContent(w io.Writer, entry *filer2.Entry, offset int for _, chunkView := range chunkViews { - urlString, err := operation.LookupFileId(fs.getMasterNode(), chunkView.FileId) + urlString, err := operation.LookupFileId(fs.filer.GetMaster(), chunkView.FileId) if err != nil { glog.V(1).Infof("operation LookupFileId %s failed, err: %v", chunkView.FileId, err) return err diff --git a/weed/server/filer_server_handlers_write.go b/weed/server/filer_server_handlers_write.go index 7d93d4485..4c2820e6b 100644 --- a/weed/server/filer_server_handlers_write.go +++ b/weed/server/filer_server_handlers_write.go @@ -32,7 +32,7 @@ func (fs *FilerServer) queryFileInfoByPath(w http.ResponseWriter, r *http.Reques writeJsonError(w, r, http.StatusInternalServerError, err) } else { fileId = entry.Chunks[0].FileId - urlLocation, err = operation.LookupFileId(fs.getMasterNode(), fileId) + urlLocation, err = operation.LookupFileId(fs.filer.GetMaster(), fileId) if err != nil { glog.V(1).Infof("operation LookupFileId %s failed, err is %s", fileId, err.Error()) w.WriteHeader(http.StatusNotFound) @@ -48,7 +48,7 @@ func (fs *FilerServer) assignNewFileInfo(w http.ResponseWriter, r *http.Request, Collection: collection, Ttl: r.URL.Query().Get("ttl"), } - assignResult, ae := operation.Assign(fs.getMasterNode(), ar) + assignResult, ae := operation.Assign(fs.filer.GetMaster(), ar) if ae != nil { glog.V(0).Infoln("failing to assign a file id", ae.Error()) writeJsonError(w, r, http.StatusInternalServerError, ae) @@ -145,7 +145,7 @@ func (fs *FilerServer) PostHandler(w http.ResponseWriter, r *http.Request) { if ret.Name != "" { path += ret.Name } else { - operation.DeleteFile(fs.getMasterNode(), fileId, fs.jwt(fileId)) //clean up + operation.DeleteFile(fs.filer.GetMaster(), fileId, fs.jwt(fileId)) //clean up glog.V(0).Infoln("Can not to write to folder", path, "without a file name!") writeJsonError(w, r, http.StatusInternalServerError, errors.New("Can not to write to folder "+path+" without a file name")) @@ -157,7 +157,7 @@ func (fs *FilerServer) PostHandler(w http.ResponseWriter, r *http.Request) { if r.Method != "PUT" { if entry, err := fs.filer.FindEntry(filer2.FullPath(path)); err == nil { oldFid := entry.Chunks[0].FileId - operation.DeleteFile(fs.getMasterNode(), oldFid, fs.jwt(oldFid)) + operation.DeleteFile(fs.filer.GetMaster(), oldFid, fs.jwt(oldFid)) } else if err != nil && err != filer2.ErrNotFound { glog.V(0).Infof("error %v occur when finding %s in filer store", err, path) } @@ -176,7 +176,7 @@ func (fs *FilerServer) PostHandler(w http.ResponseWriter, r *http.Request) { }}, } if db_err := fs.filer.CreateEntry(entry); db_err != nil { - operation.DeleteFile(fs.getMasterNode(), fileId, fs.jwt(fileId)) //clean up + operation.DeleteFile(fs.filer.GetMaster(), fileId, fs.jwt(fileId)) //clean up glog.V(0).Infof("failing to write %s to filer server : %v", path, db_err) writeJsonError(w, r, http.StatusInternalServerError, db_err) return diff --git a/weed/server/filer_server_handlers_write_autochunk.go b/weed/server/filer_server_handlers_write_autochunk.go index 5156ae02c..adc50d030 100644 --- a/weed/server/filer_server_handlers_write_autochunk.go +++ b/weed/server/filer_server_handlers_write_autochunk.go @@ -147,7 +147,7 @@ func (fs *FilerServer) doAutoChunk(w http.ResponseWriter, r *http.Request, conte if entry, err := fs.filer.FindEntry(filer2.FullPath(path)); err == nil { for _, chunk := range entry.Chunks { oldFid := chunk.FileId - operation.DeleteFile(fs.getMasterNode(), oldFid, fs.jwt(oldFid)) + operation.DeleteFile(fs.filer.GetMaster(), oldFid, fs.jwt(oldFid)) } } else if err != nil { glog.V(0).Infof("error %v occur when finding %s in filer store", err, path) diff --git a/weed/server/master_grpc_server.go b/weed/server/master_grpc_server.go index d9b8f9e09..e97cc126e 100644 --- a/weed/server/master_grpc_server.go +++ b/weed/server/master_grpc_server.go @@ -77,3 +77,15 @@ func (ms *MasterServer) SendHeartbeat(stream master_pb.Seaweed_SendHeartbeatServ } } } + +func (ms *MasterServer) KeepConnected(stream master_pb.Seaweed_KeepConnectedServer) error { + for { + _, err := stream.Recv() + if err != nil { + return err + } + if err := stream.Send(&master_pb.Empty{}); err != nil { + return err + } + } +} diff --git a/weed/server/volume_grpc_client.go b/weed/server/volume_grpc_client.go index 2f3f36924..7688745e2 100644 --- a/weed/server/volume_grpc_client.go +++ b/weed/server/volume_grpc_client.go @@ -7,49 +7,51 @@ import ( "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/pb/master_pb" "github.com/chrislusf/seaweedfs/weed/security" - "github.com/chrislusf/seaweedfs/weed/storage" "golang.org/x/net/context" "google.golang.org/grpc" ) +func (vs *VolumeServer) GetMaster() string { + return vs.currentMaster +} func (vs *VolumeServer) heartbeat() { - glog.V(0).Infof("Volume server bootstraps with master %s", vs.GetMasterNode()) - vs.masterNodes = storage.NewMasterNodes(vs.masterNode) + glog.V(0).Infof("Volume server start with masters: %v", vs.MasterNodes) vs.store.SetDataCenter(vs.dataCenter) vs.store.SetRack(vs.rack) + var err error + var newLeader string for { - err := vs.doHeartbeat(time.Duration(vs.pulseSeconds) * time.Second) - if err != nil { - glog.V(0).Infof("heartbeat error: %v", err) - time.Sleep(time.Duration(vs.pulseSeconds) * time.Second) + for _, master := range vs.MasterNodes { + if newLeader != "" { + master = newLeader + } + newLeader, err = vs.doHeartbeat(master, time.Duration(vs.pulseSeconds)*time.Second) + if err != nil { + glog.V(0).Infof("heartbeat error: %v", err) + time.Sleep(time.Duration(vs.pulseSeconds) * time.Second) + } } } } -func (vs *VolumeServer) doHeartbeat(sleepInterval time.Duration) error { - - vs.masterNodes.Reset() - masterNode, err := vs.masterNodes.FindMaster() - if err != nil { - return fmt.Errorf("No master found: %v", err) - } +func (vs *VolumeServer) doHeartbeat(masterNode string, sleepInterval time.Duration) (newLeader string, err error) { grpcConection, err := grpc.Dial(masterNode, grpc.WithInsecure()) if err != nil { - return fmt.Errorf("fail to dial: %v", err) + return "", fmt.Errorf("fail to dial: %v", err) } defer grpcConection.Close() client := master_pb.NewSeaweedClient(grpcConection) stream, err := client.SendHeartbeat(context.Background()) if err != nil { - glog.V(0).Infof("%v.SendHeartbeat(_) = _, %v", client, err) - return err + glog.V(0).Infof("SendHeartbeat to %s: %v", masterNode, err) + return "", err } - vs.SetMasterNode(masterNode) - glog.V(0).Infof("Heartbeat to %s", masterNode) + glog.V(0).Infof("Heartbeat to: %v", masterNode) + vs.currentMaster = masterNode vs.store.Client = stream defer func() { vs.store.Client = nil }() @@ -70,7 +72,8 @@ func (vs *VolumeServer) doHeartbeat(sleepInterval time.Duration) error { vs.guard.SecretKey = security.Secret(in.GetSecretKey()) } if in.GetLeader() != "" && masterNode != in.GetLeader() { - vs.masterNodes.SetPossibleLeader(in.GetLeader()) + glog.V(0).Infof("Volume Server found a new master newLeader: %v instead of %v", in.GetLeader(), masterNode) + newLeader = in.GetLeader() doneChan <- nil return } @@ -79,7 +82,7 @@ func (vs *VolumeServer) doHeartbeat(sleepInterval time.Duration) error { if err = stream.Send(vs.store.CollectHeartbeat()); err != nil { glog.V(0).Infof("Volume Server Failed to talk with master %s: %v", masterNode, err) - return err + return "", err } tickChan := time.Tick(sleepInterval) @@ -89,11 +92,10 @@ func (vs *VolumeServer) doHeartbeat(sleepInterval time.Duration) error { case <-tickChan: if err = stream.Send(vs.store.CollectHeartbeat()); err != nil { glog.V(0).Infof("Volume Server Failed to talk with master %s: %v", masterNode, err) - return err + return "", err } - case err := <-doneChan: - glog.V(0).Infof("Volume Server heart beat stops with %v", err) - return err + case <-doneChan: + return } } } diff --git a/weed/server/volume_server.go b/weed/server/volume_server.go index 0b7e09c59..9294f9bf6 100644 --- a/weed/server/volume_server.go +++ b/weed/server/volume_server.go @@ -2,22 +2,19 @@ package weed_server import ( "net/http" - "sync" - "github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/security" "github.com/chrislusf/seaweedfs/weed/storage" ) type VolumeServer struct { - masterNode string - mnLock sync.RWMutex - pulseSeconds int - dataCenter string - rack string - store *storage.Store - guard *security.Guard - masterNodes *storage.MasterNodes + MasterNodes []string + currentMaster string + pulseSeconds int + dataCenter string + rack string + store *storage.Store + guard *security.Guard needleMapKind storage.NeedleMapType FixJpgOrientation bool @@ -28,7 +25,7 @@ func NewVolumeServer(adminMux, publicMux *http.ServeMux, ip string, port int, publicUrl string, folders []string, maxCounts []int, needleMapKind storage.NeedleMapType, - masterNode string, pulseSeconds int, + masterNodes []string, pulseSeconds int, dataCenter string, rack string, whiteList []string, fixJpgOrientation bool, @@ -41,7 +38,7 @@ func NewVolumeServer(adminMux, publicMux *http.ServeMux, ip string, FixJpgOrientation: fixJpgOrientation, ReadRedirect: readRedirect, } - vs.SetMasterNode(masterNode) + vs.MasterNodes = masterNodes vs.store = storage.NewStore(port, ip, publicUrl, folders, maxCounts, vs.needleMapKind) vs.guard = security.NewGuard(whiteList, "") @@ -76,18 +73,6 @@ func NewVolumeServer(adminMux, publicMux *http.ServeMux, ip string, return vs } -func (vs *VolumeServer) GetMasterNode() string { - vs.mnLock.RLock() - defer vs.mnLock.RUnlock() - return vs.masterNode -} - -func (vs *VolumeServer) SetMasterNode(masterNode string) { - vs.mnLock.Lock() - defer vs.mnLock.Unlock() - vs.masterNode = masterNode -} - func (vs *VolumeServer) Shutdown() { glog.V(0).Infoln("Shutting down volume server...") vs.store.Close() diff --git a/weed/server/volume_server_handlers_read.go b/weed/server/volume_server_handlers_read.go index a90d4c0e2..b784dd60e 100644 --- a/weed/server/volume_server_handlers_read.go +++ b/weed/server/volume_server_handlers_read.go @@ -46,7 +46,7 @@ func (vs *VolumeServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request) w.WriteHeader(http.StatusNotFound) return } - lookupResult, err := operation.Lookup(vs.GetMasterNode(), volumeId.String()) + lookupResult, err := operation.Lookup(vs.GetMaster(), volumeId.String()) glog.V(2).Infoln("volume", volumeId, "found on", lookupResult, "error", err) if err == nil && len(lookupResult.Locations) > 0 { u, _ := url.Parse(util.NormalizeUrl(lookupResult.Locations[0].PublicUrl)) @@ -176,7 +176,7 @@ func (vs *VolumeServer) tryHandleChunkedFile(n *storage.Needle, fileName string, chunkedFileReader := &operation.ChunkedFileReader{ Manifest: chunkManifest, - Master: vs.GetMasterNode(), + Master: vs.GetMaster(), } defer chunkedFileReader.Close() if e := writeResponseContent(fileName, mType, chunkedFileReader, w, r); e != nil { diff --git a/weed/server/volume_server_handlers_ui.go b/weed/server/volume_server_handlers_ui.go index 6fc775a6d..c75c66bae 100644 --- a/weed/server/volume_server_handlers_ui.go +++ b/weed/server/volume_server_handlers_ui.go @@ -21,14 +21,14 @@ func (vs *VolumeServer) uiStatusHandler(w http.ResponseWriter, r *http.Request) } args := struct { Version string - Master string + Masters []string Volumes interface{} DiskStatuses interface{} Stats interface{} Counters *stats.ServerStats }{ util.VERSION, - vs.masterNode, + vs.MasterNodes, vs.store.Status(), ds, infos, diff --git a/weed/server/volume_server_handlers_write.go b/weed/server/volume_server_handlers_write.go index e45c2245c..3864ec903 100644 --- a/weed/server/volume_server_handlers_write.go +++ b/weed/server/volume_server_handlers_write.go @@ -31,7 +31,7 @@ func (vs *VolumeServer) PostHandler(w http.ResponseWriter, r *http.Request) { } ret := operation.UploadResult{} - size, errorStatus := topology.ReplicatedWrite(vs.GetMasterNode(), + size, errorStatus := topology.ReplicatedWrite(vs.GetMaster(), vs.store, volumeId, needle, r) httpStatus := http.StatusCreated if errorStatus != "" { @@ -80,14 +80,14 @@ func (vs *VolumeServer) DeleteHandler(w http.ResponseWriter, r *http.Request) { return } // make sure all chunks had deleted before delete manifest - if e := chunkManifest.DeleteChunks(vs.GetMasterNode()); e != nil { + if e := chunkManifest.DeleteChunks(vs.GetMaster()); e != nil { writeJsonError(w, r, http.StatusInternalServerError, fmt.Errorf("Delete chunks error: %v", e)) return } count = chunkManifest.Size } - _, err := topology.ReplicatedDelete(vs.GetMasterNode(), vs.store, volumeId, n, r) + _, err := topology.ReplicatedDelete(vs.GetMaster(), vs.store, volumeId, n, r) if err == nil { m := make(map[string]int64) diff --git a/weed/server/volume_server_ui/templates.go b/weed/server/volume_server_ui/templates.go index c3db6e92a..5f01588f4 100644 --- a/weed/server/volume_server_ui/templates.go +++ b/weed/server/volume_server_ui/templates.go @@ -72,8 +72,8 @@ var StatusTpl = template.Must(template.New("status").Funcs(funcMap).Parse(`System Stats - - + + diff --git a/weed/storage/store.go b/weed/storage/store.go index a7d8db3a1..84ed1951d 100644 --- a/weed/storage/store.go +++ b/weed/storage/store.go @@ -1,13 +1,11 @@ package storage import ( - "errors" "fmt" "strconv" "strings" "github.com/chrislusf/seaweedfs/weed/glog" - "github.com/chrislusf/seaweedfs/weed/operation" "github.com/chrislusf/seaweedfs/weed/pb/master_pb" ) @@ -15,55 +13,6 @@ const ( MAX_TTL_VOLUME_REMOVAL_DELAY = 10 // 10 minutes ) -type MasterNodes struct { - nodes []string - leader string - possibleLeader string -} - -func (mn *MasterNodes) String() string { - return fmt.Sprintf("nodes:%v, leader:%s", mn.nodes, mn.leader) -} - -func NewMasterNodes(bootstrapNode string) (mn *MasterNodes) { - mn = &MasterNodes{nodes: []string{bootstrapNode}, leader: ""} - return -} -func (mn *MasterNodes) Reset() { - if mn.leader != "" { - mn.leader = "" - glog.V(0).Infof("Resetting master nodes: %v", mn) - } -} -func (mn *MasterNodes) SetPossibleLeader(possibleLeader string) { - // TODO try to check this leader first - mn.possibleLeader = possibleLeader -} -func (mn *MasterNodes) FindMaster() (leader string, err error) { - if len(mn.nodes) == 0 { - return "", errors.New("No master node found!") - } - if mn.leader == "" { - for _, m := range mn.nodes { - glog.V(4).Infof("Listing masters on %s", m) - if leader, masters, e := operation.ListMasters(m); e == nil { - if leader != "" { - mn.nodes = append(masters, m) - mn.leader = leader - glog.V(2).Infof("current master nodes is %v", mn) - break - } - } else { - glog.V(4).Infof("Failed listing masters on %s: %v", m, e) - } - } - } - if mn.leader == "" { - return "", errors.New("No master node available!") - } - return mn.leader, nil -} - /* * A VolumeServer contains one Store */ From ebf58709d3c041084199561f4272acb0523e6736 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Fri, 1 Jun 2018 23:24:34 -0700 Subject: [PATCH 83/83] remove unused register file function --- weed/operation/filer/register.go | 36 ------------- weed/server/filer_server.go | 1 - weed/server/filer_server_handlers_admin.go | 60 ---------------------- 3 files changed, 97 deletions(-) delete mode 100644 weed/operation/filer/register.go delete mode 100644 weed/server/filer_server_handlers_admin.go diff --git a/weed/operation/filer/register.go b/weed/operation/filer/register.go deleted file mode 100644 index 655fee1ff..000000000 --- a/weed/operation/filer/register.go +++ /dev/null @@ -1,36 +0,0 @@ -package filer - -import ( - "fmt" - "net/url" - - "github.com/chrislusf/seaweedfs/weed/security" - "github.com/chrislusf/seaweedfs/weed/util" - "strconv" -) - -type SubmitResult struct { - FileName string `json:"fileName,omitempty"` - FileUrl string `json:"fileUrl,omitempty"` - Fid string `json:"fid,omitempty"` - Size uint32 `json:"size,omitempty"` - Error string `json:"error,omitempty"` -} - -func RegisterFile(filer string, path string, fileId string, fileSize int64, mime string, uid, gid int, secret security.Secret) error { - // TODO: jwt need to be used - _ = security.GenJwt(secret, fileId) - - values := make(url.Values) - values.Add("path", path) - values.Add("fileId", fileId) - values.Add("fileSize", strconv.FormatInt(fileSize, 10)) - values.Add("uid", strconv.Itoa(uid)) - values.Add("gid", strconv.Itoa(gid)) - values.Add("mime", mime) - _, err := util.Post("http://"+filer+"/admin/register", values) - if err != nil { - return fmt.Errorf("Failed to register path %s on filer %s to file id %s : %v", path, filer, fileId, err) - } - return nil -} diff --git a/weed/server/filer_server.go b/weed/server/filer_server.go index 827971a0d..6da6b5561 100644 --- a/weed/server/filer_server.go +++ b/weed/server/filer_server.go @@ -51,7 +51,6 @@ func NewFilerServer(defaultMux, readonlyMux *http.ServeMux, ip string, port int, fs.filer.LoadConfiguration() - defaultMux.HandleFunc("/admin/register", fs.registerHandler) defaultMux.HandleFunc("/favicon.ico", faviconHandler) defaultMux.HandleFunc("/", fs.filerHandler) if defaultMux != readonlyMux { diff --git a/weed/server/filer_server_handlers_admin.go b/weed/server/filer_server_handlers_admin.go deleted file mode 100644 index 93a2ab883..000000000 --- a/weed/server/filer_server_handlers_admin.go +++ /dev/null @@ -1,60 +0,0 @@ -package weed_server - -import ( - "net/http" - - "github.com/chrislusf/seaweedfs/weed/filer2" - "github.com/chrislusf/seaweedfs/weed/glog" - "github.com/chrislusf/seaweedfs/weed/pb/filer_pb" - "strconv" - "time" - "fmt" -) - -func (fs *FilerServer) registerHandler(w http.ResponseWriter, r *http.Request) { - path := r.FormValue("path") - fileId := r.FormValue("fileId") - fileSize, err := strconv.ParseUint(r.FormValue("fileSize"), 10, 64) - if err != nil { - glog.V(0).Infof("register %s to %s parse fileSize %s: %v", fileId, path, r.FormValue("fileSize"), err) - writeJsonError(w, r, http.StatusInternalServerError, fmt.Errorf("parsing fileSize: %v", err)) - return - } - uid, err := strconv.ParseUint(r.FormValue("uid"), 10, 64) - if err != nil && r.FormValue("uid") != "" { - glog.V(0).Infof("register %s to %s parse uid %s: %v", fileId, path, r.FormValue("uid"), err) - writeJsonError(w, r, http.StatusInternalServerError, fmt.Errorf("parsing uid: %v", err)) - return - } - gid, err := strconv.ParseUint(r.FormValue("gid"), 10, 64) - if err != nil && r.FormValue("gid") != "" { - glog.V(0).Infof("register %s to %s parse gid %s: %v", fileId, path, r.FormValue("gid"), err) - writeJsonError(w, r, http.StatusInternalServerError, fmt.Errorf("parsing gid: %v", err)) - return - } - mime := r.FormValue("mime") - entry := &filer2.Entry{ - FullPath: filer2.FullPath(path), - Attr: filer2.Attr{ - Mode: 0660, - Crtime: time.Now(), - Mtime: time.Now(), - Uid: uint32(uid), - Gid: uint32(gid), - Mime: mime, - }, - Chunks: []*filer_pb.FileChunk{{ - FileId: fileId, - Size: fileSize, - Mtime: time.Now().UnixNano(), - }}, - } - glog.V(2).Infof("register %s to %s parse fileSize %s", fileId, path, r.FormValue("fileSize")) - err = fs.filer.CreateEntry(entry) - if err != nil { - glog.V(0).Infof("register %s to %s error: %v", fileId, path, err) - writeJsonError(w, r, http.StatusInternalServerError, fmt.Errorf("create %s: %v", path, err)) - } else { - w.WriteHeader(http.StatusOK) - } -}
Master{{.Master}}Masters{{.Masters}}
Weekly # ReadRequests