refactor volumeSizeLimit

weed/command/master.go

@@ -37,6 +37,7 @@ var (
 	metaFolder              = cmdMaster.Flag.String("mdir", os.TempDir(), "data directory to store meta data")
 	masterPeers             = cmdMaster.Flag.String("peers", "", "all master nodes in comma separated ip:port list, example: 127.0.0.1:9093,127.0.0.1:9094")
 	volumeSizeLimitMiB      = cmdMaster.Flag.Uint("volumeSizeLimitMB", 30*1000, "Master stops directing writes to oversized volumes. (MiB)")
+	volumeSizeLimitArg      = cmdMaster.Flag.String("volumeSizeLimit", "", "Master stops directing writes to oversized volumes. (eg: 30G, 20G)")
 	volumePreallocate       = cmdMaster.Flag.Bool("volumePreallocate", false, "Preallocate disk space for volumes.")
 	mpulse                  = cmdMaster.Flag.Int("pulseSeconds", 5, "number of seconds between heartbeats")
 	defaultReplicaPlacement = cmdMaster.Flag.String("defaultReplication", "000", "Default replication type if not specified.")
@@ -64,13 +65,12 @@ func runMaster(cmd *Command, args []string) bool {
 	if *masterWhiteListOption != "" {
 		masterWhiteList = strings.Split(*masterWhiteListOption, ",")
 	}
-	if *volumeSizeLimitMiB > 30*1000 {
-		glog.Fatalf("volumeSizeLimitMB should be smaller than 30000")
-	}
+
+	volumeSizeLimit := util.ParseVolumeSizeLimit(*volumeSizeLimitMiB, *volumeSizeLimitArg)
 
 	r := mux.NewRouter()
 	ms := weed_server.NewMasterServer(r, *mport, *metaFolder,
-		*volumeSizeLimitMiB, *volumePreallocate,
+		volumeSizeLimit, *volumePreallocate,
 		*mpulse, *defaultReplicaPlacement, *garbageThreshold,
 		masterWhiteList, *masterSecureKey,
 	)

weed/command/server.go

@@ -63,7 +63,9 @@ var (
 	masterPort               = cmdServer.Flag.Int("master.port", 9333, "master server http listen port")
 	masterGrpcPort           = cmdServer.Flag.Int("master.port.grpc", 0, "master grpc server listen port, default to http port + 10000")
 	masterMetaFolder         = cmdServer.Flag.String("master.dir", "", "data directory to store meta data, default to same as -dir specified")
-	masterVolumeSizeLimitMB       = cmdServer.Flag.Uint("master.volumeSizeLimitMB", 30*1000, "Master stops directing writes to oversized volumes.")
+	masterVolumeSizeLimitMib = cmdServer.Flag.Uint("master.volumeSizeLimitMB", 30*1000, "Master stops directing writes to oversized volumes.")
+	masterVolumeSizeLimitArg = cmdMaster.Flag.String("master.volumeSizeLimit", "", "Master stops directing writes to oversized volumes. (eg: 30G, 20G)")
+
 	masterVolumePreallocate       = cmdServer.Flag.Bool("master.volumePreallocate", false, "Preallocate disk space for volumes.")
 	masterDefaultReplicaPlacement = cmdServer.Flag.String("master.defaultReplicaPlacement", "000", "Default replication type if not specified.")
 	volumeDataFolders             = cmdServer.Flag.String("dir", os.TempDir(), "directories to store data files. dir[,dir]...")
@@ -134,9 +136,7 @@ func runServer(cmd *Command, args []string) bool {
 
 	folders := strings.Split(*volumeDataFolders, ",")
 
-	if *masterVolumeSizeLimitMB > 30*1000 {
-		glog.Fatalf("masterVolumeSizeLimitMB should be less than 30000")
-	}
+	volumeSizeLimit := util.ParseVolumeSizeLimit(*masterVolumeSizeLimitMib, *masterVolumeSizeLimitArg)
 
 	if *masterMetaFolder == "" {
 		*masterMetaFolder = folders[0]
@@ -168,7 +168,7 @@ func runServer(cmd *Command, args []string) bool {
 	go func() {
 		r := mux.NewRouter()
 		ms := weed_server.NewMasterServer(r, *masterPort, *masterMetaFolder,
-			*masterVolumeSizeLimitMB, *masterVolumePreallocate,
+			volumeSizeLimit, *masterVolumePreallocate,
 			*pulseSeconds, *masterDefaultReplicaPlacement, *serverGarbageThreshold,
 			serverWhiteList, *serverSecureKey,
 		)

weed/server/master_grpc_server.go

@@ -64,7 +64,7 @@ func (ms *MasterServer) SendHeartbeat(stream master_pb.Seaweed_SendHeartbeatServ
 			dn = rack.GetOrCreateDataNode(heartbeat.Ip, int(heartbeat.Port), heartbeat.PublicUrl, int(heartbeat.MaxVolumeCount))
 			glog.V(0).Infof("added volume server %v:%d", heartbeat.GetIp(), heartbeat.GetPort())
 			if err := stream.Send(&master_pb.HeartbeatResponse{
-				VolumeSizeLimit: uint64(ms.volumeSizeLimitMiB) * 1024 * 1024,
+				VolumeSizeLimit: ms.volumeSizeLimit,
 				SecretKey:       string(ms.guard.SecretKey),
 			}); err != nil {
 				return err

weed/server/master_server.go

@@ -20,7 +20,7 @@ import (
 type MasterServer struct {
 	port                    int
 	metaFolder              string
-	volumeSizeLimitMiB      uint
+	volumeSizeLimit         uint64
 	preallocate             int64
 	pulseSeconds            int
 	defaultReplicaPlacement string
@@ -39,7 +39,7 @@ type MasterServer struct {
 }
 
 func NewMasterServer(r *mux.Router, port int, metaFolder string,
-	volumeSizeLimitMiB uint,
+	volumeSizeLimit uint64,
 	preallocate bool,
 	pulseSeconds int,
 	defaultReplicaPlacement string,
@@ -50,11 +50,11 @@ func NewMasterServer(r *mux.Router, port int, metaFolder string,
 
 	var preallocateSize int64
 	if preallocate {
-		preallocateSize = int64(volumeSizeLimitMiB) * (1 << 20)
+		preallocateSize = int64(volumeSizeLimit) * (1 << 20)
 	}
 	ms := &MasterServer{
 		port:                    port,
-		volumeSizeLimitMiB:      volumeSizeLimitMiB,
+		volumeSizeLimit:         volumeSizeLimit,
 		preallocate:             preallocateSize,
 		pulseSeconds:            pulseSeconds,
 		defaultReplicaPlacement: defaultReplicaPlacement,
@@ -63,9 +63,9 @@ func NewMasterServer(r *mux.Router, port int, metaFolder string,
 	}
 	ms.bounedLeaderChan = make(chan int, 16)
 	seq := sequence.NewMemorySequencer()
-	ms.Topo = topology.NewTopology("topo", seq, uint64(volumeSizeLimitMiB)*1024*1024, pulseSeconds)
+	ms.Topo = topology.NewTopology("topo", seq, uint64(volumeSizeLimit), pulseSeconds)
 	ms.vg = topology.NewDefaultVolumeGrowth()
-	glog.V(0).Infoln("Volume Size Limit is", volumeSizeLimitMiB, "MiB")
+	glog.V(0).Infoln("Volume Size Limit is", volumeSizeLimit)
 
 	ms.guard = security.NewGuard(whiteList, secureKey)
 

weed/util/arg.go

@@ -0,0 +1,22 @@
+package util
+
+import (
+	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/dustin/go-humanize"
+)
+
+func ParseVolumeSizeLimit(volumeSizeLimitMiBValue uint, volumeSizeLimitArgValue string) uint64 {
+	volumeSizeLimit := uint64(volumeSizeLimitMiBValue) * 1024 * 1024
+	if volumeSizeLimitArgValue != "" {
+		var err error
+		volumeSizeLimit, err = humanize.ParseBytes(volumeSizeLimitArgValue)
+		if err != nil {
+			glog.Fatalf("Parse volumeSizeLimit %s : %s", volumeSizeLimitMiBValue, err)
+		}
+	}
+	if volumeSizeLimit > uint64(30*1000)*1024*1024 {
+		glog.Fatalf("volumeSizeLimitMB should be smaller than 30000")
+	}
+
+	return volumeSizeLimit
+}