From 46755ea1e1e4d79e4c0368aa7fcb17ee8aa812cc Mon Sep 17 00:00:00 2001 From: Lei Liu Date: Tue, 12 Nov 2019 14:21:43 +0800 Subject: [PATCH] fix master maintenance logic Signed-off-by: Lei Liu --- .../remove_duplicate_fids.go | 6 ++--- weed/command/scaffold.go | 2 ++ weed/command/shell.go | 24 +------------------ weed/operation/assign_file_id.go | 14 +++++------ weed/server/master_server.go | 20 +++++++++++----- .../backend/memory_map/memory_map_backend.go | 4 ++-- weed/util/config.go | 1 - weed/util/parse.go | 21 ++++++++++++++++ 8 files changed, 50 insertions(+), 42 deletions(-) diff --git a/unmaintained/remove_duplicate_fids/remove_duplicate_fids.go b/unmaintained/remove_duplicate_fids/remove_duplicate_fids.go index d44bc5354..a162d1757 100644 --- a/unmaintained/remove_duplicate_fids/remove_duplicate_fids.go +++ b/unmaintained/remove_duplicate_fids/remove_duplicate_fids.go @@ -26,9 +26,9 @@ type VolumeFileScanner4SeeDat struct { version needle.Version block storage.SuperBlock - dir string - hashes map[string]bool - dat *os.File + dir string + hashes map[string]bool + dat *os.File datBackend backend.DataStorageBackend } diff --git a/weed/command/scaffold.go b/weed/command/scaffold.go index 6fa72c730..76b244a05 100644 --- a/weed/command/scaffold.go +++ b/weed/command/scaffold.go @@ -346,6 +346,8 @@ scripts = """ """ sleep_minutes = 17 # sleep minutes between each script execution +filer_url = "http://localhost:8888/" + sequencer_type = memory # Choose [memory|etcd] type for storing the file id sequence # when sequencer.type = etcd, set listen client urls of etcd cluster that store file id sequence diff --git a/weed/command/shell.go b/weed/command/shell.go index 91aa8770a..34b5aef31 100644 --- a/weed/command/shell.go +++ b/weed/command/shell.go @@ -2,9 +2,6 @@ package command import ( "fmt" - "net/url" - "strconv" - "strings" "github.com/chrislusf/seaweedfs/weed/security" "github.com/chrislusf/seaweedfs/weed/shell" @@ -37,7 +34,7 @@ func runShell(command *Command, args []string) bool { shellOptions.GrpcDialOption = security.LoadClientTLS(viper.Sub("grpc"), "client") var filerPwdErr error - shellOptions.FilerHost, shellOptions.FilerPort, shellOptions.Directory, filerPwdErr = parseFilerUrl(*shellInitialFilerUrl) + shellOptions.FilerHost, shellOptions.FilerPort, shellOptions.Directory, filerPwdErr = util.ParseFilerUrl(*shellInitialFilerUrl) if filerPwdErr != nil { fmt.Printf("failed to parse url filer.url=%s : %v\n", *shellInitialFilerUrl, filerPwdErr) return false @@ -48,22 +45,3 @@ func runShell(command *Command, args []string) bool { return true } - -func parseFilerUrl(entryPath string) (filerServer string, filerPort int64, path string, err error) { - if !strings.HasPrefix(entryPath, "http://") && !strings.HasPrefix(entryPath, "https://") { - entryPath = "http://" + entryPath - } - - var u *url.URL - u, err = url.Parse(entryPath) - if err != nil { - return - } - filerServer = u.Hostname() - portString := u.Port() - if portString != "" { - filerPort, err = strconv.ParseInt(portString, 10, 32) - } - path = u.Path - return -} diff --git a/weed/operation/assign_file_id.go b/weed/operation/assign_file_id.go index 2971cddbb..2dfa44483 100644 --- a/weed/operation/assign_file_id.go +++ b/weed/operation/assign_file_id.go @@ -47,13 +47,13 @@ func Assign(server string, grpcDialOption grpc.DialOption, primaryRequest *Volum lastError = WithMasterServerClient(server, grpcDialOption, func(masterClient master_pb.SeaweedClient) error { req := &master_pb.AssignRequest{ - Count: primaryRequest.Count, - Replication: primaryRequest.Replication, - Collection: primaryRequest.Collection, - Ttl: primaryRequest.Ttl, - DataCenter: primaryRequest.DataCenter, - Rack: primaryRequest.Rack, - DataNode: primaryRequest.DataNode, + Count: primaryRequest.Count, + Replication: primaryRequest.Replication, + Collection: primaryRequest.Collection, + Ttl: primaryRequest.Ttl, + DataCenter: primaryRequest.DataCenter, + Rack: primaryRequest.Rack, + DataNode: primaryRequest.DataNode, WritableVolumeCount: primaryRequest.WritableVolumeCount, } resp, grpcErr := masterClient.Assign(context.Background(), req) diff --git a/weed/server/master_server.go b/weed/server/master_server.go index 15e6ee51c..e68356c01 100644 --- a/weed/server/master_server.go +++ b/weed/server/master_server.go @@ -182,16 +182,21 @@ func (ms *MasterServer) proxyToLeader(f func(w http.ResponseWriter, r *http.Requ } func (ms *MasterServer) startAdminScripts() { + var err error + v := viper.GetViper() adminScripts := v.GetString("master.maintenance.scripts") - v.SetDefault("master.maintenance.sleep_minutes", 17) - sleepMinutes := v.GetInt("master.maintenance.sleep_minutes") - glog.V(0).Infof("adminScripts:\n%v", adminScripts) if adminScripts == "" { return } + v.SetDefault("master.maintenance.sleep_minutes", 17) + sleepMinutes := v.GetInt("master.maintenance.sleep_minutes") + + v.SetDefault("master.maintenance.filer_url", "http://localhost:8888/") + filerURL := v.GetString("master.maintenance.filer_url") + scriptLines := strings.Split(adminScripts, "\n") masterAddress := "localhost:" + strconv.Itoa(ms.option.Port) @@ -199,9 +204,12 @@ func (ms *MasterServer) startAdminScripts() { var shellOptions shell.ShellOptions shellOptions.GrpcDialOption = security.LoadClientTLS(viper.Sub("grpc"), "master") shellOptions.Masters = &masterAddress - shellOptions.FilerHost = "localhost" - shellOptions.FilerPort = 8888 - shellOptions.Directory = "/" + + shellOptions.FilerHost, shellOptions.FilerPort, shellOptions.Directory, err = util.ParseFilerUrl(filerURL) + if err != nil { + glog.V(0).Infof("failed to parse master.maintenance.filer_url=%s : %v\n", filerURL, err) + return + } commandEnv := shell.NewCommandEnv(shellOptions) diff --git a/weed/storage/backend/memory_map/memory_map_backend.go b/weed/storage/backend/memory_map/memory_map_backend.go index d999b917e..3f475d79f 100644 --- a/weed/storage/backend/memory_map/memory_map_backend.go +++ b/weed/storage/backend/memory_map/memory_map_backend.go @@ -17,14 +17,14 @@ type MemoryMappedFile struct { func NewMemoryMappedFile(f *os.File, memoryMapSizeMB uint32) *MemoryMappedFile { mmf := &MemoryMappedFile{ - mm : new(MemoryMap), + mm: new(MemoryMap), } mmf.mm.CreateMemoryMap(f, 1024*1024*uint64(memoryMapSizeMB)) return mmf } func (mmf *MemoryMappedFile) ReadAt(p []byte, off int64) (n int, err error) { - readBytes, e := mmf.mm.ReadMemory(uint64(off), uint64(len(p))) + readBytes, e := mmf.mm.ReadMemory(uint64(off), uint64(len(p))) if e != nil { return 0, e } diff --git a/weed/util/config.go b/weed/util/config.go index 7e2f9b373..0ace53a37 100644 --- a/weed/util/config.go +++ b/weed/util/config.go @@ -45,4 +45,3 @@ func LoadConfiguration(configFileName string, required bool) (loaded bool) { func Config() Configuration { return viper.GetViper() } - diff --git a/weed/util/parse.go b/weed/util/parse.go index 0a8317c19..6593d43b6 100644 --- a/weed/util/parse.go +++ b/weed/util/parse.go @@ -1,7 +1,9 @@ package util import ( + "net/url" "strconv" + "strings" ) func ParseInt(text string, defaultValue int) int { @@ -24,3 +26,22 @@ func ParseUint64(text string, defaultValue uint64) uint64 { } return count } + +func ParseFilerUrl(entryPath string) (filerServer string, filerPort int64, path string, err error) { + if !strings.HasPrefix(entryPath, "http://") && !strings.HasPrefix(entryPath, "https://") { + entryPath = "http://" + entryPath + } + + var u *url.URL + u, err = url.Parse(entryPath) + if err != nil { + return + } + filerServer = u.Hostname() + portString := u.Port() + if portString != "" { + filerPort, err = strconv.ParseInt(portString, 10, 32) + } + path = u.Path + return +}