Merge pull request #1115 from iliul/enhanced-master-logic

fix master maintenance logic

unmaintained/remove_duplicate_fids/remove_duplicate_fids.go

@@ -26,9 +26,9 @@ type VolumeFileScanner4SeeDat struct {
 	version needle.Version
 	block   storage.SuperBlock
 
-	dir    string
-	hashes map[string]bool
-	dat    *os.File
+	dir        string
+	hashes     map[string]bool
+	dat        *os.File
 	datBackend backend.DataStorageBackend
 }
 

weed/command/scaffold.go

@@ -346,6 +346,8 @@ scripts = """
 """
 sleep_minutes = 17          # sleep minutes between each script execution
 
+filer_url = "http://localhost:8888/"
+
 sequencer_type = memory     # Choose [memory|etcd] type for storing the file id sequence
 
 # when sequencer.type = etcd, set listen client urls of etcd cluster that store file id sequence

weed/command/shell.go

@@ -2,9 +2,6 @@ package command
 
 import (
 	"fmt"
-	"net/url"
-	"strconv"
-	"strings"
 
 	"github.com/chrislusf/seaweedfs/weed/security"
 	"github.com/chrislusf/seaweedfs/weed/shell"
@@ -37,7 +34,7 @@ func runShell(command *Command, args []string) bool {
 	shellOptions.GrpcDialOption = security.LoadClientTLS(viper.Sub("grpc"), "client")
 
 	var filerPwdErr error
-	shellOptions.FilerHost, shellOptions.FilerPort, shellOptions.Directory, filerPwdErr = parseFilerUrl(*shellInitialFilerUrl)
+	shellOptions.FilerHost, shellOptions.FilerPort, shellOptions.Directory, filerPwdErr = util.ParseFilerUrl(*shellInitialFilerUrl)
 	if filerPwdErr != nil {
 		fmt.Printf("failed to parse url filer.url=%s : %v\n", *shellInitialFilerUrl, filerPwdErr)
 		return false
@@ -48,22 +45,3 @@ func runShell(command *Command, args []string) bool {
 	return true
 
 }
-
-func parseFilerUrl(entryPath string) (filerServer string, filerPort int64, path string, err error) {
-	if !strings.HasPrefix(entryPath, "http://") && !strings.HasPrefix(entryPath, "https://") {
-		entryPath = "http://" + entryPath
-	}
-
-	var u *url.URL
-	u, err = url.Parse(entryPath)
-	if err != nil {
-		return
-	}
-	filerServer = u.Hostname()
-	portString := u.Port()
-	if portString != "" {
-		filerPort, err = strconv.ParseInt(portString, 10, 32)
-	}
-	path = u.Path
-	return
-}

weed/operation/assign_file_id.go

@@ -47,13 +47,13 @@ func Assign(server string, grpcDialOption grpc.DialOption, primaryRequest *Volum
 		lastError = WithMasterServerClient(server, grpcDialOption, func(masterClient master_pb.SeaweedClient) error {
 
 			req := &master_pb.AssignRequest{
-				Count:       primaryRequest.Count,
-				Replication: primaryRequest.Replication,
-				Collection:  primaryRequest.Collection,
-				Ttl:         primaryRequest.Ttl,
-				DataCenter:  primaryRequest.DataCenter,
-				Rack:        primaryRequest.Rack,
-				DataNode:    primaryRequest.DataNode,
+				Count:               primaryRequest.Count,
+				Replication:         primaryRequest.Replication,
+				Collection:          primaryRequest.Collection,
+				Ttl:                 primaryRequest.Ttl,
+				DataCenter:          primaryRequest.DataCenter,
+				Rack:                primaryRequest.Rack,
+				DataNode:            primaryRequest.DataNode,
 				WritableVolumeCount: primaryRequest.WritableVolumeCount,
 			}
 			resp, grpcErr := masterClient.Assign(context.Background(), req)

weed/server/master_server.go

@@ -182,16 +182,21 @@ func (ms *MasterServer) proxyToLeader(f func(w http.ResponseWriter, r *http.Requ
 }
 
 func (ms *MasterServer) startAdminScripts() {
+	var err error
+
 	v := viper.GetViper()
 	adminScripts := v.GetString("master.maintenance.scripts")
-	v.SetDefault("master.maintenance.sleep_minutes", 17)
-	sleepMinutes := v.GetInt("master.maintenance.sleep_minutes")
-
 	glog.V(0).Infof("adminScripts:\n%v", adminScripts)
 	if adminScripts == "" {
 		return
 	}
 
+	v.SetDefault("master.maintenance.sleep_minutes", 17)
+	sleepMinutes := v.GetInt("master.maintenance.sleep_minutes")
+
+	v.SetDefault("master.maintenance.filer_url", "http://localhost:8888/")
+	filerURL := v.GetString("master.maintenance.filer_url")
+
 	scriptLines := strings.Split(adminScripts, "\n")
 
 	masterAddress := "localhost:" + strconv.Itoa(ms.option.Port)
@@ -199,9 +204,12 @@ func (ms *MasterServer) startAdminScripts() {
 	var shellOptions shell.ShellOptions
 	shellOptions.GrpcDialOption = security.LoadClientTLS(viper.Sub("grpc"), "master")
 	shellOptions.Masters = &masterAddress
-	shellOptions.FilerHost = "localhost"
-	shellOptions.FilerPort = 8888
-	shellOptions.Directory = "/"
+
+	shellOptions.FilerHost, shellOptions.FilerPort, shellOptions.Directory, err = util.ParseFilerUrl(filerURL)
+	if err != nil {
+		glog.V(0).Infof("failed to parse master.maintenance.filer_url=%s : %v\n", filerURL, err)
+		return
+	}
 
 	commandEnv := shell.NewCommandEnv(shellOptions)
 

weed/storage/backend/memory_map/memory_map_backend.go

@@ -17,14 +17,14 @@ type MemoryMappedFile struct {
 
 func NewMemoryMappedFile(f *os.File, memoryMapSizeMB uint32) *MemoryMappedFile {
 	mmf := &MemoryMappedFile{
-		mm : new(MemoryMap),
+		mm: new(MemoryMap),
 	}
 	mmf.mm.CreateMemoryMap(f, 1024*1024*uint64(memoryMapSizeMB))
 	return mmf
 }
 
 func (mmf *MemoryMappedFile) ReadAt(p []byte, off int64) (n int, err error) {
-	readBytes, e :=  mmf.mm.ReadMemory(uint64(off), uint64(len(p)))
+	readBytes, e := mmf.mm.ReadMemory(uint64(off), uint64(len(p)))
 	if e != nil {
 		return 0, e
 	}

weed/util/config.go

@@ -45,4 +45,3 @@ func LoadConfiguration(configFileName string, required bool) (loaded bool) {
 func Config() Configuration {
 	return viper.GetViper()
 }
-

weed/util/parse.go

@@ -1,7 +1,9 @@
 package util
 
 import (
+	"net/url"
 	"strconv"
+	"strings"
 )
 
 func ParseInt(text string, defaultValue int) int {
@@ -24,3 +26,22 @@ func ParseUint64(text string, defaultValue uint64) uint64 {
 	}
 	return count
 }
+
+func ParseFilerUrl(entryPath string) (filerServer string, filerPort int64, path string, err error) {
+	if !strings.HasPrefix(entryPath, "http://") && !strings.HasPrefix(entryPath, "https://") {
+		entryPath = "http://" + entryPath
+	}
+
+	var u *url.URL
+	u, err = url.Parse(entryPath)
+	if err != nil {
+		return
+	}
+	filerServer = u.Hostname()
+	portString := u.Port()
+	if portString != "" {
+		filerPort, err = strconv.ParseInt(portString, 10, 32)
+	}
+	path = u.Path
+	return
+}