From c63cebaee11b03c44ead1d4563c22329ffd99220 Mon Sep 17 00:00:00 2001 From: cschiano Date: Mon, 18 Jul 2016 10:28:24 +0200 Subject: [PATCH 01/11] Add a visualization for the filer --- weed/server/filer_server_handlers_read.go | 74 ++++++----------------- weed/server/filer_ui/templates.go | 53 ++++++++++++++++ 2 files changed, 72 insertions(+), 55 deletions(-) create mode 100644 weed/server/filer_ui/templates.go diff --git a/weed/server/filer_server_handlers_read.go b/weed/server/filer_server_handlers_read.go index 8340021ce..1470bcc48 100644 --- a/weed/server/filer_server_handlers_read.go +++ b/weed/server/filer_server_handlers_read.go @@ -1,15 +1,12 @@ package weed_server import ( - "io" "net/http" - "net/url" "strconv" "strings" "github.com/chrislusf/seaweedfs/weed/glog" - "github.com/chrislusf/seaweedfs/weed/operation" - "github.com/chrislusf/seaweedfs/weed/util" + ui "github.com/chrislusf/seaweedfs/weed/server/filer_ui" "github.com/syndtr/goleveldb/leveldb" ) @@ -42,63 +39,30 @@ func (fs *FilerServer) listDirectoryHandler(w http.ResponseWriter, r *http.Reque } func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request, isGetMethod bool) { - if strings.HasSuffix(r.URL.Path, "/") { - if fs.disableDirListing { - w.WriteHeader(http.StatusMethodNotAllowed) - return - } - fs.listDirectoryHandler(w, r) - return - } + fileLimit := 100 + files, err := fs.filer.ListFiles(r.URL.Path, "", fileLimit) - fileId, err := fs.filer.FindFile(r.URL.Path) if err == leveldb.ErrNotFound { - glog.V(3).Infoln("Not found in db", r.URL.Path) - w.WriteHeader(http.StatusNotFound) + glog.V(0).Infof("Error %s", err) return } - urlLocation, err := operation.LookupFileId(fs.getMasterNode(), fileId) - if err != nil { - glog.V(1).Infoln("operation LookupFileId %s failed, err is %s", fileId, err.Error()) - w.WriteHeader(http.StatusNotFound) - return - } - urlString := urlLocation - if fs.redirectOnRead { - http.Redirect(w, r, urlString, http.StatusFound) + directories, err2 := fs.filer.ListDirectories(r.URL.Path) + if err2 == leveldb.ErrNotFound { + glog.V(0).Infof("Error %s", err) return } - u, _ := url.Parse(urlString) - q := u.Query() - for key, values := range r.URL.Query() { - for _, value := range values { - q.Add(key, value) - } - } - u.RawQuery = q.Encode() - request := &http.Request{ - Method: r.Method, - URL: u, - Proto: r.Proto, - ProtoMajor: r.ProtoMajor, - ProtoMinor: r.ProtoMinor, - Header: r.Header, - Body: r.Body, - Host: r.Host, - ContentLength: r.ContentLength, - } - glog.V(3).Infoln("retrieving from", u) - resp, do_err := util.Do(request) - if do_err != nil { - glog.V(0).Infoln("failing to connect to volume server", do_err.Error()) - writeJsonError(w, r, http.StatusInternalServerError, do_err) - return - } - defer resp.Body.Close() - for k, v := range resp.Header { - w.Header()[k] = v + + args := struct { + Path string + Files interface{} + Directories interface{} + NotAllFilesDisplayed bool + }{ + r.URL.Path, + files, + directories, + len(files) == fileLimit, } - w.WriteHeader(resp.StatusCode) - io.Copy(w, resp.Body) + ui.StatusTpl.Execute(w, args) } diff --git a/weed/server/filer_ui/templates.go b/weed/server/filer_ui/templates.go new file mode 100644 index 000000000..b4d16d4db --- /dev/null +++ b/weed/server/filer_ui/templates.go @@ -0,0 +1,53 @@ +package master_ui + +import ( + "html/template" +) + +var StatusTpl = template.Must(template.New("status").Parse(` + + + SeaweedFS Filer + + + + +
+
+

+ + SeaweedFS Filer +

+
+
+ {{.Path}} +
+ +
+
    + {{$path := .Path }} + {{ range $dirs_index, $dir := .Directories }} +
  • + + {{ $dir.Name }} + +
    • + {{ end }} + + {{ range $file_index, $file := .Files }} +
  • + {{ $file.Name }} +
    • + {{ end }} +
+
+ + {{if .NotAllFilesDisplayed}} +
+ Not all files are displayed. +
+ {{end}} +
+ + +`)) From af905a3ff719118bdde900f4c1387f738fbb83dd Mon Sep 17 00:00:00 2001 From: cschiano Date: Wed, 20 Jul 2016 10:46:28 +0200 Subject: [PATCH 02/11] Add limit parameter and pagination for files --- weed/server/filer_server_handlers_read.go | 59 +++++++++++++---------- weed/server/filer_ui/templates.go | 6 ++- 2 files changed, 37 insertions(+), 28 deletions(-) diff --git a/weed/server/filer_server_handlers_read.go b/weed/server/filer_server_handlers_read.go index 1470bcc48..98ec13f95 100644 --- a/weed/server/filer_server_handlers_read.go +++ b/weed/server/filer_server_handlers_read.go @@ -10,59 +10,66 @@ import ( "github.com/syndtr/goleveldb/leveldb" ) -// listDirectoryHandler lists directories and folers under a directory -// files are sorted by name and paginated via "lastFileName" and "limit". -// sub directories are listed on the first page, when "lastFileName" -// is empty. -func (fs *FilerServer) listDirectoryHandler(w http.ResponseWriter, r *http.Request) { +func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request, isGetMethod bool) { if !strings.HasSuffix(r.URL.Path, "/") { return } - dirlist, err := fs.filer.ListDirectories(r.URL.Path) - if err == leveldb.ErrNotFound { - glog.V(3).Infoln("Directory Not Found in db", r.URL.Path) - w.WriteHeader(http.StatusNotFound) + + if fs.disableDirListing { + w.WriteHeader(http.StatusMethodNotAllowed) return } - m := make(map[string]interface{}) - m["Directory"] = r.URL.Path - lastFileName := r.FormValue("lastFileName") - if lastFileName == "" { - m["Subdirectories"] = dirlist - } + limit, limit_err := strconv.Atoi(r.FormValue("limit")) if limit_err != nil { limit = 100 } - m["Files"], _ = fs.filer.ListFiles(r.URL.Path, lastFileName, limit) - writeJsonQuiet(w, r, http.StatusOK, m) -} -func (fs *FilerServer) GetOrHeadHandler(w http.ResponseWriter, r *http.Request, isGetMethod bool) { - fileLimit := 100 - files, err := fs.filer.ListFiles(r.URL.Path, "", fileLimit) + lastFileName := r.FormValue("lastFileName") + files, err := fs.filer.ListFiles(r.URL.Path, lastFileName, limit) if err == leveldb.ErrNotFound { glog.V(0).Infof("Error %s", err) + w.WriteHeader(http.StatusNotFound) return } directories, err2 := fs.filer.ListDirectories(r.URL.Path) if err2 == leveldb.ErrNotFound { glog.V(0).Infof("Error %s", err) + w.WriteHeader(http.StatusNotFound) return } + shouldDisplayLoadMore := len(files) > 0 + + lastFileName = "" + if len(files) > 0 { + lastFileName = files[len(files)-1].Name + + files2, err3 := fs.filer.ListFiles(r.URL.Path, lastFileName, limit) + if err3 == leveldb.ErrNotFound { + glog.V(0).Infof("Error %s", err) + w.WriteHeader(http.StatusNotFound) + return + } + shouldDisplayLoadMore = len(files2) > 0 + } + args := struct { - Path string - Files interface{} - Directories interface{} - NotAllFilesDisplayed bool + Path string + Files interface{} + Directories interface{} + Limit int + LastFileName string + ShouldDisplayLoadMore bool }{ r.URL.Path, files, directories, - len(files) == fileLimit, + limit, + lastFileName, + shouldDisplayLoadMore, } ui.StatusTpl.Execute(w, args) } diff --git a/weed/server/filer_ui/templates.go b/weed/server/filer_ui/templates.go index b4d16d4db..6f4f7ce86 100644 --- a/weed/server/filer_ui/templates.go +++ b/weed/server/filer_ui/templates.go @@ -42,9 +42,11 @@ var StatusTpl = template.Must(template.New("status").Parse(` - {{if .NotAllFilesDisplayed}} +{{if .ShouldDisplayLoadMore}}
- Not all files are displayed. + Date: Wed, 20 Jul 2016 23:45:55 -0700 Subject: [PATCH 04/11] add "weed copy" command to copy files to filer --- weed/command/command.go | 1 + weed/command/copy.go | 147 +++++++++++++++++++++ weed/command/server.go | 3 +- weed/command/upload.go | 10 +- weed/operation/filer/register.go | 31 +++++ weed/server/filer_server.go | 12 +- weed/server/filer_server_handlers_admin.go | 12 ++ weed/util/http_util.go | 11 +- 8 files changed, 217 insertions(+), 10 deletions(-) create mode 100644 weed/command/copy.go create mode 100644 weed/operation/filer/register.go diff --git a/weed/command/command.go b/weed/command/command.go index d654f57cd..c451936e5 100644 --- a/weed/command/command.go +++ b/weed/command/command.go @@ -11,6 +11,7 @@ var Commands = []*Command{ cmdBenchmark, cmdBackup, cmdCompact, + cmdCopy, cmdFix, cmdServer, cmdMaster, diff --git a/weed/command/copy.go b/weed/command/copy.go new file mode 100644 index 000000000..0e7109daf --- /dev/null +++ b/weed/command/copy.go @@ -0,0 +1,147 @@ +package command + +import ( + "fmt" + "io/ioutil" + "net/url" + "os" + "path/filepath" + "strings" + + "github.com/chrislusf/seaweedfs/weed/operation" + filer_operation "github.com/chrislusf/seaweedfs/weed/operation/filer" + "github.com/chrislusf/seaweedfs/weed/security" +) + +var ( + copy CopyOptions +) + +type CopyOptions struct { + master *string + include *string + replication *string + collection *string + ttl *string + maxMB *int + secretKey *string + + secret security.Secret +} + +func init() { + cmdCopy.Run = runCopy // break init cycle + cmdCopy.IsDebug = cmdCopy.Flag.Bool("debug", false, "verbose debug information") + copy.master = cmdCopy.Flag.String("master", "localhost:9333", "SeaweedFS master location") + copy.include = cmdCopy.Flag.String("include", "", "pattens of files to copy, e.g., *.pdf, *.html, ab?d.txt, works together with -dir") + copy.replication = cmdCopy.Flag.String("replication", "", "replication type") + copy.collection = cmdCopy.Flag.String("collection", "", "optional collection name") + copy.ttl = cmdCopy.Flag.String("ttl", "", "time to live, e.g.: 1m, 1h, 1d, 1M, 1y") + copy.maxMB = cmdCopy.Flag.Int("maxMB", 0, "split files larger than the limit") + copy.secretKey = cmdCopy.Flag.String("secure.secret", "", "secret to encrypt Json Web Token(JWT)") +} + +var cmdCopy = &Command{ + UsageLine: "copy file_or_dir1 [file_or_dir2 file_or_dir3] http://localhost:8888/path/to/a/folder/", + Short: "copy one or a list of files to a filer folder", + Long: `copy one or a list of files, or batch copy one whole folder recursively, to a filer folder + + It can copy one or a list of files or folders. + + If copying a whole folder recursively: + All files under the folder and subfolders will be copyed. + Optional parameter "-include" allows you to specify the file name patterns. + + If any file has a ".gz" extension, the content are considered gzipped already, and will be stored as is. + This can save volume server's gzipped processing and allow customizable gzip compression level. + The file name will strip out ".gz" and stored. For example, "jquery.js.gz" will be stored as "jquery.js". + + If "maxMB" is set to a positive number, files larger than it would be split into chunks and copyed separatedly. + The list of file ids of those chunks would be stored in an additional chunk, and this additional chunk's file id would be returned. + + `, +} + +func runCopy(cmd *Command, args []string) bool { + copy.secret = security.Secret(*copy.secretKey) + if len(args) <= 1 { + return false + } + filerDestination := args[len(args)-1] + fileOrDirs := args[0 : len(args)-1] + + filerUrl, err := url.Parse(filerDestination) + if err != nil { + fmt.Printf("The last argument should be a URL on filer: %v\n", err) + return false + } + if len(fileOrDirs) > 1 && !strings.HasSuffix(filerUrl.Path, "/") { + fmt.Println("Can not copy multiple items to a file. The last argument must be a folder ended with \"/\"") + return false + } + + for _, fileOrDir := range fileOrDirs { + if !doEachCopy(fileOrDir, filerUrl.Host, filerUrl.Path) { + return false + } + } + return true +} + +func doEachCopy(fileOrDir string, host string, path string) bool { + f, err := os.Open(fileOrDir) + if err != nil { + fmt.Printf("Failed to open file %s: %v", fileOrDir, err) + return false + } + defer f.Close() + + fi, err := f.Stat() + if err != nil { + fmt.Printf("Failed to get stat for file %s: %v", fileOrDir, err) + return false + } + + mode := fi.Mode() + if mode.IsDir() { + files, _ := ioutil.ReadDir(fileOrDir) + for _, subFileOrDir := range files { + if !doEachCopy(fileOrDir+"/"+subFileOrDir.Name(), host, path+fi.Name()+"/") { + return false + } + } + return true + } + + // this is a regular file + if *copy.include != "" { + if ok, _ := filepath.Match(*copy.include, filepath.Base(fileOrDir)); !ok { + return true + } + } + + parts, err := operation.NewFileParts([]string{fileOrDir}) + if err != nil { + fmt.Printf("Failed to read file %s: %v", fileOrDir, err) + } + + results, err := operation.SubmitFiles(*copy.master, parts, + *copy.replication, *copy.collection, + *copy.ttl, *copy.maxMB, copy.secret) + if err != nil { + fmt.Printf("Failed to submit file %s: %v", fileOrDir, err) + } + + if strings.HasSuffix(path, "/") { + path = path + fi.Name() + } + + if err = filer_operation.RegisterFile(host, path, results[0].Fid, copy.secret); err != nil { + fmt.Printf("Failed to register file %s on %s: %v", fileOrDir, host, err) + return false + } + + fmt.Printf("Copy %s => http://%s/%s\n", fileOrDir, host, path) + + return true +} diff --git a/weed/command/server.go b/weed/command/server.go index 6ed1e5228..1211c7137 100644 --- a/weed/command/server.go +++ b/weed/command/server.go @@ -82,7 +82,7 @@ func init() { filerOptions.master = cmdServer.Flag.String("filer.master", "", "default to current master server") filerOptions.collection = cmdServer.Flag.String("filer.collection", "", "all data will be stored in this collection") filerOptions.port = cmdServer.Flag.Int("filer.port", 8888, "filer server http listen port") - filerOptions.dir = cmdServer.Flag.String("filer.dir", "", "directory to store meta data, default to a 'filer' sub directory of what -mdir is specified") + filerOptions.dir = cmdServer.Flag.String("filer.dir", "", "directory to store meta data, default to a 'filer' sub directory of what -dir is specified") filerOptions.defaultReplicaPlacement = cmdServer.Flag.String("filer.defaultReplicaPlacement", "", "Default replication type if not specified during runtime.") filerOptions.redirectOnRead = cmdServer.Flag.Bool("filer.redirectOnRead", false, "whether proxy or redirect to volume server during file GET request") filerOptions.disableDirListing = cmdServer.Flag.Bool("filer.disableDirListing", false, "turn off directory listing") @@ -164,6 +164,7 @@ func runServer(cmd *Command, args []string) bool { if *isStartingFiler { go func() { + time.Sleep(1 * time.Second) r := http.NewServeMux() _, nfs_err := weed_server.NewFilerServer(r, *serverBindIp, *filerOptions.port, *filerOptions.master, *filerOptions.dir, *filerOptions.collection, *filerOptions.defaultReplicaPlacement, diff --git a/weed/command/upload.go b/weed/command/upload.go index 0dfa115bb..1f0696f70 100644 --- a/weed/command/upload.go +++ b/weed/command/upload.go @@ -15,7 +15,7 @@ var ( ) type UploadOptions struct { - server *string + master *string dir *string include *string replication *string @@ -28,7 +28,7 @@ type UploadOptions struct { func init() { cmdUpload.Run = runUpload // break init cycle cmdUpload.IsDebug = cmdUpload.Flag.Bool("debug", false, "verbose debug information") - upload.server = cmdUpload.Flag.String("server", "localhost:9333", "SeaweedFS master location") + upload.master = cmdUpload.Flag.String("master", "localhost:9333", "SeaweedFS master location") upload.dir = cmdUpload.Flag.String("dir", "", "Upload the whole folder recursively if specified.") upload.include = cmdUpload.Flag.String("include", "", "pattens of files to upload, e.g., *.pdf, *.html, ab?d.txt, works together with -dir") upload.replication = cmdUpload.Flag.String("replication", "", "replication type") @@ -39,7 +39,7 @@ func init() { } var cmdUpload = &Command{ - UsageLine: "upload -server=localhost:9333 file1 [file2 file3]\n weed upload -server=localhost:9333 -dir=one_directory -include=*.pdf", + UsageLine: "upload -master=localhost:9333 file1 [file2 file3]\n weed upload -server=localhost:9333 -dir=one_directory -include=*.pdf", Short: "upload one or a list of files", Long: `upload one or a list of files, or batch upload one whole folder recursively. @@ -79,7 +79,7 @@ func runUpload(cmd *Command, args []string) bool { if e != nil { return e } - results, e := operation.SubmitFiles(*upload.server, parts, + results, e := operation.SubmitFiles(*upload.master, parts, *upload.replication, *upload.collection, *upload.ttl, *upload.maxMB, secret) bytes, _ := json.Marshal(results) @@ -98,7 +98,7 @@ func runUpload(cmd *Command, args []string) bool { if e != nil { fmt.Println(e.Error()) } - results, _ := operation.SubmitFiles(*upload.server, parts, + results, _ := operation.SubmitFiles(*upload.master, parts, *upload.replication, *upload.collection, *upload.ttl, *upload.maxMB, secret) bytes, _ := json.Marshal(results) diff --git a/weed/operation/filer/register.go b/weed/operation/filer/register.go new file mode 100644 index 000000000..2c4703680 --- /dev/null +++ b/weed/operation/filer/register.go @@ -0,0 +1,31 @@ +package operation + +import ( + "fmt" + "net/url" + + "github.com/chrislusf/seaweedfs/weed/security" + "github.com/chrislusf/seaweedfs/weed/util" +) + +type SubmitResult struct { + FileName string `json:"fileName,omitempty"` + FileUrl string `json:"fileUrl,omitempty"` + Fid string `json:"fid,omitempty"` + Size uint32 `json:"size,omitempty"` + Error string `json:"error,omitempty"` +} + +func RegisterFile(filer string, path string, fileId string, secret security.Secret) error { + // TODO: jwt need to be used + _ = security.GenJwt(secret, fileId) + + values := make(url.Values) + values.Add("path", path) + values.Add("fileId", fileId) + _, err := util.Post("http://"+filer+"/admin/register", values) + if err != nil { + return fmt.Errorf("Failed to register path:%s on filer:%s to file id:%s", path, filer, fileId) + } + return nil +} diff --git a/weed/server/filer_server.go b/weed/server/filer_server.go index 1b54f0840..b99bbd7c9 100644 --- a/weed/server/filer_server.go +++ b/weed/server/filer_server.go @@ -62,6 +62,7 @@ func NewFilerServer(r *http.ServeMux, ip string, port int, master string, dir st } r.HandleFunc("/admin/mv", fs.moveHandler) + r.HandleFunc("/admin/register", fs.registerHandler) } r.HandleFunc("/", fs.filerHandler) @@ -73,9 +74,14 @@ func NewFilerServer(r *http.ServeMux, ip string, port int, master string, dir st glog.V(0).Infof("Filer server bootstraps with master %s", fs.getMasterNode()) //force initialize with all available master nodes - _, err := fs.masterNodes.FindMaster() - if err != nil { - glog.Fatalf("filer server failed to get master cluster info:%s", err.Error()) + for { + _, err := fs.masterNodes.FindMaster() + if err != nil { + glog.Infof("filer server failed to get master cluster info:%s", err.Error()) + time.Sleep(3 * time.Second) + } else { + break + } } for { diff --git a/weed/server/filer_server_handlers_admin.go b/weed/server/filer_server_handlers_admin.go index 979ad517b..aa7c09986 100644 --- a/weed/server/filer_server_handlers_admin.go +++ b/weed/server/filer_server_handlers_admin.go @@ -27,3 +27,15 @@ func (fs *FilerServer) moveHandler(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) } } + +func (fs *FilerServer) registerHandler(w http.ResponseWriter, r *http.Request) { + path := r.FormValue("path") + fileId := r.FormValue("fileId") + err := fs.filer.CreateFile(path, fileId) + if err != nil { + glog.V(4).Infof("register %s to %s error: %v", fileId, path, err) + writeJsonError(w, r, http.StatusInternalServerError, err) + } else { + w.WriteHeader(http.StatusOK) + } +} diff --git a/weed/util/http_util.go b/weed/util/http_util.go index a54fc8779..2379e3b2b 100644 --- a/weed/util/http_util.go +++ b/weed/util/http_util.go @@ -32,6 +32,9 @@ func PostBytes(url string, body []byte) ([]byte, error) { return nil, fmt.Errorf("Post to %s: %v", url, err) } defer r.Body.Close() + if r.StatusCode >= 400 { + return nil, fmt.Errorf("%s: %s", url, r.Status) + } b, err := ioutil.ReadAll(r.Body) if err != nil { return nil, fmt.Errorf("Read response body: %v", err) @@ -45,6 +48,9 @@ func Post(url string, values url.Values) ([]byte, error) { return nil, err } defer r.Body.Close() + if r.StatusCode >= 400 { + return nil, fmt.Errorf("%s: %s", url, r.Status) + } b, err := ioutil.ReadAll(r.Body) if err != nil { return nil, err @@ -59,7 +65,7 @@ func Get(url string) ([]byte, error) { } defer r.Body.Close() b, err := ioutil.ReadAll(r.Body) - if r.StatusCode != 200 { + if r.StatusCode >= 400 { return nil, fmt.Errorf("%s: %s", url, r.Status) } if err != nil { @@ -81,6 +87,9 @@ func Delete(url string, jwt security.EncodedJwt) error { return e } defer resp.Body.Close() + if resp.StatusCode >= 400 { + return fmt.Errorf("%s: %s", url, resp.Status) + } body, err := ioutil.ReadAll(resp.Body) if err != nil { return err From a57162e8bf14b5b25bd246bfc01e725bdccf76f3 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Thu, 21 Jul 2016 00:40:13 -0700 Subject: [PATCH 05/11] delete operation does not need this checking --- weed/util/http_util.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/weed/util/http_util.go b/weed/util/http_util.go index 2379e3b2b..83302663e 100644 --- a/weed/util/http_util.go +++ b/weed/util/http_util.go @@ -87,9 +87,6 @@ func Delete(url string, jwt security.EncodedJwt) error { return e } defer resp.Body.Close() - if resp.StatusCode >= 400 { - return fmt.Errorf("%s: %s", url, resp.Status) - } body, err := ioutil.ReadAll(resp.Body) if err != nil { return err From a5be4a6d40dee779f390bd75c1d5480cd1167e22 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Thu, 21 Jul 2016 01:23:56 -0700 Subject: [PATCH 06/11] fix package name --- weed/operation/filer/register.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/weed/operation/filer/register.go b/weed/operation/filer/register.go index 2c4703680..875616f26 100644 --- a/weed/operation/filer/register.go +++ b/weed/operation/filer/register.go @@ -1,4 +1,4 @@ -package operation +package filer import ( "fmt" From 185a916f5eae022b89cc310d895fd93cac2fa3eb Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Thu, 21 Jul 2016 15:00:07 -0700 Subject: [PATCH 07/11] adjusting command options --- weed/command/{copy.go => filer_copy.go} | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) rename weed/command/{copy.go => filer_copy.go} (90%) diff --git a/weed/command/copy.go b/weed/command/filer_copy.go similarity index 90% rename from weed/command/copy.go rename to weed/command/filer_copy.go index 0e7109daf..2aa994f6f 100644 --- a/weed/command/copy.go +++ b/weed/command/filer_copy.go @@ -42,7 +42,7 @@ func init() { } var cmdCopy = &Command{ - UsageLine: "copy file_or_dir1 [file_or_dir2 file_or_dir3] http://localhost:8888/path/to/a/folder/", + UsageLine: "filer.copy file_or_dir1 [file_or_dir2 file_or_dir3] http://localhost:8888/path/to/a/folder/", Short: "copy one or a list of files to a filer folder", Long: `copy one or a list of files, or batch copy one whole folder recursively, to a filer folder @@ -75,13 +75,13 @@ func runCopy(cmd *Command, args []string) bool { fmt.Printf("The last argument should be a URL on filer: %v\n", err) return false } - if len(fileOrDirs) > 1 && !strings.HasSuffix(filerUrl.Path, "/") { - fmt.Println("Can not copy multiple items to a file. The last argument must be a folder ended with \"/\"") - return false + path := filerUrl.Path + if !strings.HasSuffix(path, "/") { + path = path + "/" } for _, fileOrDir := range fileOrDirs { - if !doEachCopy(fileOrDir, filerUrl.Host, filerUrl.Path) { + if !doEachCopy(fileOrDir, filerUrl.Host, path) { return false } } @@ -141,7 +141,7 @@ func doEachCopy(fileOrDir string, host string, path string) bool { return false } - fmt.Printf("Copy %s => http://%s/%s\n", fileOrDir, host, path) + fmt.Printf("Copy %s => http://%s%s\n", fileOrDir, host, path) return true } From 78678f4bcb7aa6b18cd364f86135b54e2d633f24 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=9C=8D=E6=99=93=E6=A0=8B?= Date: Mon, 25 Jul 2016 11:40:35 +0800 Subject: [PATCH 08/11] deleted needle does not need checksum verification --- weed/storage/needle_read_write.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/weed/storage/needle_read_write.go b/weed/storage/needle_read_write.go index 2f26147d6..3ac236951 100644 --- a/weed/storage/needle_read_write.go +++ b/weed/storage/needle_read_write.go @@ -158,6 +158,9 @@ func (n *Needle) ReadData(r *os.File, offset int64, size uint32, version Version case Version2: n.readNeedleDataVersion2(bytes[NeedleHeaderSize : NeedleHeaderSize+int(n.Size)]) } + if size == 0 { + return nil + } checksum := util.BytesToUint32(bytes[NeedleHeaderSize+size : NeedleHeaderSize+size+NeedleChecksumSize]) newChecksum := NewCRC(n.Data) if checksum != newChecksum.Value() { From 09bd3d015d02fb9a8c237d64586a0fbc8a6bde8f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=9C=8D=E6=99=93=E6=A0=8B?= Date: Mon, 25 Jul 2016 14:54:40 +0800 Subject: [PATCH 09/11] deleted index entry could not point to deleted needle --- weed/storage/volume_checking.go | 4 ++++ weed/topology/volume_layout.go | 1 - 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/weed/storage/volume_checking.go b/weed/storage/volume_checking.go index 00ce0aab1..5fcaf3b66 100644 --- a/weed/storage/volume_checking.go +++ b/weed/storage/volume_checking.go @@ -21,6 +21,10 @@ func CheckVolumeDataIntegrity(v *Volume, indexFile *os.File) error { return fmt.Errorf("readLastIndexEntry %s failed: %v", indexFile.Name(), e) } key, offset, size := idxFileEntry(lastIdxEntry) + //deleted index entry could not point to deleted needle + if offset == 0 || size == 0 { + return nil + } if e = verifyNeedleIntegrity(v.dataFile, v.Version(), int64(offset)*NeedlePaddingSize, key, size); e != nil { return fmt.Errorf("verifyNeedleIntegrity %s failed: %v", indexFile.Name(), e) } diff --git a/weed/topology/volume_layout.go b/weed/topology/volume_layout.go index 066f5f69a..afb14b6e4 100644 --- a/weed/topology/volume_layout.go +++ b/weed/topology/volume_layout.go @@ -43,7 +43,6 @@ func (vl *VolumeLayout) RegisterVolume(v *storage.VolumeInfo, dn *DataNode) { if _, ok := vl.vid2location[v.Id]; !ok { vl.vid2location[v.Id] = NewVolumeLocationList() } - vl.vid2location[v.Id].Set(dn) glog.V(4).Infoln("volume", v.Id, "added to dn", dn.Id(), "len", vl.vid2location[v.Id].Length(), "copy", v.ReplicaPlacement.GetCopyCount()) if vl.vid2location[v.Id].Length() == vl.rp.GetCopyCount() && vl.isWritable(v) { if _, ok := vl.oversizedVolumes[v.Id]; !ok { From b9b3651a98d3ba19565ded8ccaa8d4beb5c79f17 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=9C=8D=E6=99=93=E6=A0=8B?= Date: Mon, 25 Jul 2016 14:56:58 +0800 Subject: [PATCH 10/11] deleted index entry could not point to deleted needle --- weed/storage/volume_checking.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/weed/storage/volume_checking.go b/weed/storage/volume_checking.go index 5fcaf3b66..d424010f1 100644 --- a/weed/storage/volume_checking.go +++ b/weed/storage/volume_checking.go @@ -22,7 +22,7 @@ func CheckVolumeDataIntegrity(v *Volume, indexFile *os.File) error { } key, offset, size := idxFileEntry(lastIdxEntry) //deleted index entry could not point to deleted needle - if offset == 0 || size == 0 { + if offset == 0 { return nil } if e = verifyNeedleIntegrity(v.dataFile, v.Version(), int64(offset)*NeedlePaddingSize, key, size); e != nil { From 52e55508da33db3632b9c2976aaec04b4b9fb4bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=9C=8D=E6=99=93=E6=A0=8B?= Date: Mon, 25 Jul 2016 15:07:11 +0800 Subject: [PATCH 11/11] deleted index entry could not point to deleted needle --- weed/topology/volume_layout.go | 1 + 1 file changed, 1 insertion(+) diff --git a/weed/topology/volume_layout.go b/weed/topology/volume_layout.go index afb14b6e4..066f5f69a 100644 --- a/weed/topology/volume_layout.go +++ b/weed/topology/volume_layout.go @@ -43,6 +43,7 @@ func (vl *VolumeLayout) RegisterVolume(v *storage.VolumeInfo, dn *DataNode) { if _, ok := vl.vid2location[v.Id]; !ok { vl.vid2location[v.Id] = NewVolumeLocationList() } + vl.vid2location[v.Id].Set(dn) glog.V(4).Infoln("volume", v.Id, "added to dn", dn.Id(), "len", vl.vid2location[v.Id].Length(), "copy", v.ReplicaPlacement.GetCopyCount()) if vl.vid2location[v.Id].Length() == vl.rp.GetCopyCount() && vl.isWritable(v) { if _, ok := vl.oversizedVolumes[v.Id]; !ok {