minFreeSpace refactored

unmaintained/repeated_vacuum/repeated_vacuum.go

@@ -52,7 +52,7 @@ func main() {
 }
 
 func genFile(grpcDialOption grpc.DialOption, i int) (*operation.AssignResult, string) {
-	assignResult, err := operation.Assign(*master, grpcDialOption, &operation.VolumeAssignRequest{
+	assignResult, err := operation.Assign(func() string { return *master }, grpcDialOption, &operation.VolumeAssignRequest{
 		Count:       1,
 		Replication: *replication,
 	})

weed/command/server.go

@@ -58,7 +58,8 @@ var (
 	serverDisableHttp         = cmdServer.Flag.Bool("disableHttp", false, "disable http requests, only gRPC operations are allowed.")
 	volumeDataFolders         = cmdServer.Flag.String("dir", os.TempDir(), "directories to store data files. dir[,dir]...")
 	volumeMaxDataVolumeCounts = cmdServer.Flag.String("volume.max", "8", "maximum numbers of volumes, count[,count]... If set to zero, the limit will be auto configured.")
-	volumeMinFreeSpacePercent = cmdServer.Flag.String("volume.minFreeSpacePercent", "1", "minimum free disk space (default to 1%). Low disk space will mark all volumes as ReadOnly.")
+	volumeMinFreeSpacePercent = cmdServer.Flag.String("volume.minFreeSpacePercent", "1", "minimum free disk space (default to 1%). Low disk space will mark all volumes as ReadOnly (deprecated, use minFreeSpace instead).")
+	volumeMinFreeSpace        = cmdServer.Flag.String("volume.minFreeSpace", "", "min free disk space (value<=100 as percentage like 1, other as human readable bytes, like 10GiB). Low disk space will mark all volumes as ReadOnly.")
 	serverMetricsHttpPort     = cmdServer.Flag.Int("metricsPort", 0, "Prometheus metrics listen port")
 
 	// pulseSeconds              = cmdServer.Flag.Int("pulseSeconds", 5, "number of seconds between heartbeats")
@@ -244,8 +245,8 @@ func runServer(cmd *Command, args []string) bool {
 
 	// start volume server
 	if *isStartingVolumeServer {
-		go serverOptions.v.startVolumeServer(*volumeDataFolders, *volumeMaxDataVolumeCounts, *serverWhiteListOption, *volumeMinFreeSpacePercent)
-
+		minFreeSpaces := util.MustParseMinFreeSpace(*minFreeSpace, *minFreeSpacePercent)
+		go serverOptions.v.startVolumeServer(*volumeDataFolders, *volumeMaxDataVolumeCounts, *serverWhiteListOption, minFreeSpaces)
 	}
 
 	if *isStartingMasterServer {
@@ -253,6 +254,4 @@ func runServer(cmd *Command, args []string) bool {
 	}
 
 	select {}
-
-	return true
 }

weed/command/volume.go

@@ -57,7 +57,6 @@ type VolumeServerOptions struct {
 	compactionMBPerSecond   *int
 	fileSizeLimitMB         *int
 	concurrentUploadLimitMB *int
-	minFreeSpaces           []float32
 	pprof                   *bool
 	preStopSeconds          *int
 	metricsHttpPort         *int
@@ -121,13 +120,13 @@ func runVolume(cmd *Command, args []string) bool {
 
 	go stats_collect.StartMetricsServer(*v.metricsHttpPort)
 
-	v.startVolumeServer(*volumeFolders, *maxVolumeCounts, *volumeWhiteListOption,
-		util.EmptyTo(*minFreeSpace, *minFreeSpacePercent))
+	minFreeSpaces := util.MustParseMinFreeSpace(*minFreeSpace, *minFreeSpacePercent)
+	v.startVolumeServer(*volumeFolders, *maxVolumeCounts, *volumeWhiteListOption, minFreeSpaces)
 
 	return true
 }
 
-func (v VolumeServerOptions) startVolumeServer(volumeFolders, maxVolumeCounts, volumeWhiteListOption, minFreeSpace string) {
+func (v VolumeServerOptions) startVolumeServer(volumeFolders, maxVolumeCounts, volumeWhiteListOption string, minFreeSpaces []util.MinFreeSpace) {
 
 	// Set multiple folders and each folder's max volume count limit'
 	v.folders = strings.Split(volumeFolders, ",")
@@ -155,22 +154,13 @@ func (v VolumeServerOptions) startVolumeServer(volumeFolders, maxVolumeCounts, v
 		glog.Fatalf("%d directories by -dir, but only %d max is set by -max", len(v.folders), len(v.folderMaxLimits))
 	}
 
-	// set minFreeSpacePercent
-	minFreeSpaceStrings := strings.Split(minFreeSpace, ",")
-	for _, freeString := range minFreeSpaceStrings {
-		if vv, e := util.ParseMinFreeSpace(freeString); e == nil {
-			v.minFreeSpaces = append(v.minFreeSpaces, vv)
-		} else {
-			glog.Fatalf("The value specified in -minFreeSpace not a valid value %s", freeString)
-		}
-	}
-	if len(v.minFreeSpaces) == 1 && len(v.folders) > 1 {
+	if len(minFreeSpaces) == 1 && len(v.folders) > 1 {
 		for i := 0; i < len(v.folders)-1; i++ {
-			v.minFreeSpaces = append(v.minFreeSpaces, v.minFreeSpaces[0])
+			minFreeSpaces = append(minFreeSpaces, minFreeSpaces[0])
 		}
 	}
-	if len(v.folders) != len(v.minFreeSpaces) {
-		glog.Fatalf("%d directories by -dir, but only %d minFreeSpacePercent is set by -minFreeSpacePercent", len(v.folders), len(v.minFreeSpaces))
+	if len(v.folders) != len(minFreeSpaces) {
+		glog.Fatalf("%d directories by -dir, but only %d minFreeSpacePercent is set by -minFreeSpacePercent", len(v.folders), len(minFreeSpaces))
 	}
 
 	// set disk types
@@ -233,7 +223,7 @@ func (v VolumeServerOptions) startVolumeServer(volumeFolders, maxVolumeCounts, v
 
 	volumeServer := weed_server.NewVolumeServer(volumeMux, publicVolumeMux,
 		*v.ip, *v.port, *v.publicUrl,
-		v.folders, v.folderMaxLimits, v.minFreeSpaces, diskTypes,
+		v.folders, v.folderMaxLimits, minFreeSpaces, diskTypes,
 		*v.idxFolder,
 		volumeNeedleMapKind,
 		strings.Split(masters, ","), 5, *v.dataCenter, *v.rack,

weed/server/volume_server.go

@@ -43,7 +43,7 @@ type VolumeServer struct {
 
 func NewVolumeServer(adminMux, publicMux *http.ServeMux, ip string,
 	port int, publicUrl string,
-	folders []string, maxCounts []int, minFreeSpaces []float32, diskTypes []types.DiskType,
+	folders []string, maxCounts []int, minFreeSpaces []util.MinFreeSpace, diskTypes []types.DiskType,
 	idxFolder string,
 	needleMapKind storage.NeedleMapKind,
 	masterNodes []string, pulseSeconds int,

weed/storage/disk_location.go

@@ -23,10 +23,9 @@ type DiskLocation struct {
 	DiskType               types.DiskType
 	MaxVolumeCount         int
 	OriginalMaxVolumeCount int
-	// MinFreeSpace limits the minimum free space (<=100 as percentage, > 100 as bytes)
-	MinFreeSpace float32
-	volumes      map[needle.VolumeId]*Volume
-	volumesLock  sync.RWMutex
+	MinFreeSpace           util.MinFreeSpace
+	volumes                map[needle.VolumeId]*Volume
+	volumesLock            sync.RWMutex
 
 	// erasure coding
 	ecVolumes     map[needle.VolumeId]*erasure_coding.EcVolume
@@ -35,7 +34,7 @@ type DiskLocation struct {
 	isDiskSpaceLow bool
 }
 
-func NewDiskLocation(dir string, maxVolumeCount int, minFreeSpace float32, idxDir string, diskType types.DiskType) *DiskLocation {
+func NewDiskLocation(dir string, maxVolumeCount int, minFreeSpace util.MinFreeSpace, idxDir string, diskType types.DiskType) *DiskLocation {
 	dir = util.ResolvePath(dir)
 	if idxDir == "" {
 		idxDir = dir
@@ -363,7 +362,7 @@ func (l *DiskLocation) CheckDiskSpace() {
 			stats.VolumeServerResourceGauge.WithLabelValues(l.Directory, "used").Set(float64(s.Used))
 			stats.VolumeServerResourceGauge.WithLabelValues(l.Directory, "free").Set(float64(s.Free))
 
-			isLow := l.MinFreeSpace < 100 && s.PercentFree < l.MinFreeSpace || s.Free < uint64(l.MinFreeSpace)
+			isLow, desc := l.MinFreeSpace.IsLow(s.Free, s.PercentFree)
 			if isLow != l.isDiskSpaceLow {
 				l.isDiskSpaceLow = !l.isDiskSpaceLow
 			}
@@ -373,8 +372,7 @@ func (l *DiskLocation) CheckDiskSpace() {
 				logLevel = glog.Level(0)
 			}
 
-			glog.V(logLevel).Infof("dir %s freePercent %.2f%% < min %.2f%%, isLowDiskSpace: %v",
-				dir, s.PercentFree, l.MinFreeSpace, l.isDiskSpaceLow)
+			glog.V(logLevel).Infof("dir %s %s", dir, desc)
 		}
 		time.Sleep(time.Minute)
 	}

weed/storage/store.go

@@ -2,6 +2,7 @@ package storage
 
 import (
 	"fmt"
+	"github.com/chrislusf/seaweedfs/weed/util"
 	"path/filepath"
 	"strings"
 	"sync/atomic"
@@ -52,7 +53,8 @@ func (s *Store) String() (str string) {
 	return
 }
 
-func NewStore(grpcDialOption grpc.DialOption, port int, ip, publicUrl string, dirnames []string, maxVolumeCounts []int, minFreeSpaces []float32, idxFolder string, needleMapKind NeedleMapKind, diskTypes []DiskType) (s *Store) {
+func NewStore(grpcDialOption grpc.DialOption, port int, ip, publicUrl string, dirnames []string, maxVolumeCounts []int,
+	minFreeSpaces []util.MinFreeSpace, idxFolder string, needleMapKind NeedleMapKind, diskTypes []DiskType) (s *Store) {
 	s = &Store{grpcDialOption: grpcDialOption, Port: port, Ip: ip, PublicUrl: publicUrl, NeedleMapKind: needleMapKind}
 	s.Locations = make([]*DiskLocation, 0)
 	for i := 0; i < len(dirnames); i++ {

weed/util/bytes.go

@@ -5,7 +5,6 @@ import (
 	"crypto/md5"
 	"crypto/rand"
 	"encoding/base64"
-	"errors"
 	"fmt"
 	"io"
 	"math"
@@ -176,23 +175,12 @@ func EmptyTo(s, to string) string {
 	return s
 }
 
-var ErrMinFreeSpaceBadValue = errors.New("minFreeSpace is invalid")
-
-// ParseMinFreeSpace parses min free space expression s as percentage like 1,10 or human readable size like 10G
-func ParseMinFreeSpace(s string) (float32, error) {
-	if value, e := strconv.ParseFloat(s, 32); e == nil {
-		if value < 0 || value > 100 {
-			return 0, ErrMinFreeSpaceBadValue
-		}
-		return float32(value), nil
-	} else if directSize, e2 := ParseBytes(s); e2 == nil {
-		if directSize <= 100 {
-			return 0, ErrMinFreeSpaceBadValue
-		}
-		return float32(directSize), nil
+// IfElse works like b ? this : that.
+func IfElse(b bool, this, that string) string {
+	if b {
+		return this
 	}
-
-	return 0, ErrMinFreeSpaceBadValue
+	return that
 }
 
 // ParseBytes parses a string representation of bytes into the number

weed/util/bytes_test.go

@@ -2,32 +2,6 @@ package util
 
 import "testing"
 
-func TestParseMinFreeSpace(t *testing.T) {
-	tests := []struct {
-		in    string
-		ok    bool
-		value float32
-	}{
-		{in: "42", ok: true, value: 42},
-		{in: "-1", ok: false, value: 0},
-		{in: "101", ok: false, value: 0},
-		{in: "100B", ok: false, value: 0},
-		{in: "100Ki", ok: true, value: 100 * 1024},
-		{in: "100GiB", ok: true, value: 100 * 1024 * 1024 * 1024},
-		{in: "42M", ok: true, value: 42 * 1000 * 1000},
-	}
-
-	for _, p := range tests {
-		got, err := ParseMinFreeSpace(p.in)
-		if p.ok != (err == nil) {
-			t.Errorf("failed to test %v", p.in)
-		}
-		if p.ok && err == nil && got != p.value {
-			t.Errorf("failed to test %v", p.in)
-		}
-	}
-}
-
 func TestByteParsing(t *testing.T) {
 	tests := []struct {
 		in  string

weed/util/minfreespace.go

@@ -0,0 +1,90 @@
+package util
+
+import (
+	"errors"
+	"fmt"
+	"github.com/chrislusf/seaweedfs/weed/glog"
+	"strconv"
+	"strings"
+)
+
+// MinFreeSpaceType is the type of MinFreeSpace.
+type MinFreeSpaceType int
+
+const (
+	// AsPercent set the MinFreeSpaceType to a percentage value from 0 to 100.
+	AsPercent MinFreeSpaceType = iota
+	// AsBytes set the MinFreeSpaceType to a absolute value bytes.
+	AsBytes
+)
+
+// MinFreeSpace is type that defines the limit for the minimum free space.
+type MinFreeSpace struct {
+	Type    MinFreeSpaceType
+	Bytes   uint64
+	Percent float32
+	Raw     string
+}
+
+// IsLow tells whether the free space is low or not.
+func (s MinFreeSpace) IsLow(freeBytes uint64, freePercent float32) (yes bool, desc string) {
+	switch s.Type {
+	case AsPercent:
+		yes = freePercent < s.Percent
+		op := IfElse(yes, "<", ">=")
+		return yes, fmt.Sprintf("disk free %.2f%% %s required %.2f%%", freePercent, op, s.Percent)
+	case AsBytes:
+		yes = freeBytes < s.Bytes
+		op := IfElse(yes, "<", ">=")
+		return yes, fmt.Sprintf("disk free %s %s required %s",
+			BytesToHumanReadable(freeBytes), op, BytesToHumanReadable(s.Bytes))
+	}
+
+	return false, ""
+}
+
+// String returns a string representation of MinFreeSpace.
+func (s MinFreeSpace) String() string {
+	switch s.Type {
+	case AsPercent:
+		return fmt.Sprintf("%.2f%%", s.Percent)
+	default:
+		return s.Raw
+	}
+}
+
+// MustParseMinFreeSpace parses comma-separated argument for min free space setting.
+// minFreeSpace has the high priority than minFreeSpacePercent if it is set.
+func MustParseMinFreeSpace(minFreeSpace string, minFreeSpacePercent string) (spaces []MinFreeSpace) {
+	ss := strings.Split(EmptyTo(minFreeSpace, minFreeSpacePercent), ",")
+	for _, freeString := range ss {
+		if vv, e := ParseMinFreeSpace(freeString); e == nil {
+			spaces = append(spaces, *vv)
+		} else {
+			glog.Fatalf("The value specified in -minFreeSpace not a valid value %s", freeString)
+		}
+	}
+
+	return spaces
+}
+
+var ErrMinFreeSpaceBadValue = errors.New("minFreeSpace is invalid")
+
+// ParseMinFreeSpace parses min free space expression s as percentage like 1,10 or human readable size like 10G
+func ParseMinFreeSpace(s string) (*MinFreeSpace, error) {
+	if percent, e := strconv.ParseFloat(s, 32); e == nil {
+		if percent < 0 || percent > 100 {
+			return nil, ErrMinFreeSpaceBadValue
+		}
+		return &MinFreeSpace{Type: AsPercent, Percent: float32(percent), Raw: s}, nil
+	}
+
+	if directSize, e := ParseBytes(s); e == nil {
+		if directSize <= 100 {
+			return nil, ErrMinFreeSpaceBadValue
+		}
+		return &MinFreeSpace{Type: AsBytes, Bytes: directSize, Raw: s}, nil
+	}
+
+	return nil, ErrMinFreeSpaceBadValue
+}

weed/util/minfreespace_test.go

@@ -0,0 +1,29 @@
+package util
+
+import "testing"
+
+func TestParseMinFreeSpace(t *testing.T) {
+	tests := []struct {
+		in    string
+		ok    bool
+		value *MinFreeSpace
+	}{
+		{in: "42", ok: true, value: &MinFreeSpace{Type: AsPercent, Percent: 42, Raw: "42"}},
+		{in: "-1", ok: false, value: nil},
+		{in: "101", ok: false, value: nil},
+		{in: "100B", ok: false, value: nil},
+		{in: "100Ki", ok: true, value: &MinFreeSpace{Type: AsBytes, Bytes: 100 * 1024, Raw: "100Ki"}},
+		{in: "100GiB", ok: true, value: &MinFreeSpace{Type: AsBytes, Bytes: 100 * 1024 * 1024 * 1024, Raw: "100GiB"}},
+		{in: "42M", ok: true, value: &MinFreeSpace{Type: AsBytes, Bytes: 42 * 1000 * 1000, Raw: "42M"}},
+	}
+
+	for _, p := range tests {
+		got, err := ParseMinFreeSpace(p.in)
+		if p.ok != (err == nil) {
+			t.Errorf("failed to test %v", p.in)
+		}
+		if p.ok && err == nil && *got != *p.value {
+			t.Errorf("failed to test %v", p.in)
+		}
+	}
+}