From 31f1cdeac281fb88a3d03743f9796f81e1d74378 Mon Sep 17 00:00:00 2001 From: bingoohuang Date: Mon, 26 Apr 2021 18:48:34 +0800 Subject: [PATCH] minFreeSpace argument allows size like 10GiB --- weed/command/volume.go | 30 +++++----- weed/server/volume_server.go | 4 +- weed/storage/disk_location.go | 24 +++++--- weed/storage/store.go | 4 +- weed/util/bytes.go | 107 ++++++++++++++++++++++++++++++++++ weed/util/bytes_test.go | 85 +++++++++++++++++++++++++++ 6 files changed, 227 insertions(+), 27 deletions(-) create mode 100644 weed/util/bytes_test.go diff --git a/weed/command/volume.go b/weed/command/volume.go index 9df500178..93a19102d 100644 --- a/weed/command/volume.go +++ b/weed/command/volume.go @@ -57,7 +57,7 @@ type VolumeServerOptions struct { compactionMBPerSecond *int fileSizeLimitMB *int concurrentUploadLimitMB *int - minFreeSpacePercents []float32 + minFreeSpaces []float32 pprof *bool preStopSeconds *int metricsHttpPort *int @@ -105,7 +105,8 @@ var ( volumeFolders = cmdVolume.Flag.String("dir", os.TempDir(), "directories to store data files. dir[,dir]...") maxVolumeCounts = cmdVolume.Flag.String("max", "8", "maximum numbers of volumes, count[,count]... If set to zero, the limit will be auto configured.") volumeWhiteListOption = cmdVolume.Flag.String("whiteList", "", "comma separated Ip addresses having write permission. No limit if empty.") - minFreeSpacePercent = cmdVolume.Flag.String("minFreeSpacePercent", "1", "minimum free disk space (default to 1%). Low disk space will mark all volumes as ReadOnly.") + minFreeSpacePercent = cmdVolume.Flag.String("minFreeSpacePercent", "1", "minimum free disk space (default to 1%). Low disk space will mark all volumes as ReadOnly (deprecated, use minFreeSpace instead).") + minFreeSpace = cmdVolume.Flag.String("minFreeSpace", "", "min free disk space (value<=100 as percentage like 1, other as human readable bytes, like 10GiB). Low disk space will mark all volumes as ReadOnly.") ) func runVolume(cmd *Command, args []string) bool { @@ -120,12 +121,13 @@ func runVolume(cmd *Command, args []string) bool { go stats_collect.StartMetricsServer(*v.metricsHttpPort) - v.startVolumeServer(*volumeFolders, *maxVolumeCounts, *volumeWhiteListOption, *minFreeSpacePercent) + v.startVolumeServer(*volumeFolders, *maxVolumeCounts, *volumeWhiteListOption, + util.EmptyTo(*minFreeSpace, *minFreeSpacePercent)) return true } -func (v VolumeServerOptions) startVolumeServer(volumeFolders, maxVolumeCounts, volumeWhiteListOption, minFreeSpacePercent string) { +func (v VolumeServerOptions) startVolumeServer(volumeFolders, maxVolumeCounts, volumeWhiteListOption, minFreeSpace string) { // Set multiple folders and each folder's max volume count limit' v.folders = strings.Split(volumeFolders, ",") @@ -154,21 +156,21 @@ func (v VolumeServerOptions) startVolumeServer(volumeFolders, maxVolumeCounts, v } // set minFreeSpacePercent - minFreeSpacePercentStrings := strings.Split(minFreeSpacePercent, ",") - for _, freeString := range minFreeSpacePercentStrings { - if value, e := strconv.ParseFloat(freeString, 32); e == nil { - v.minFreeSpacePercents = append(v.minFreeSpacePercents, float32(value)) + minFreeSpaceStrings := strings.Split(minFreeSpace, ",") + for _, freeString := range minFreeSpaceStrings { + if vv, e := util.ParseMinFreeSpace(freeString); e == nil { + v.minFreeSpaces = append(v.minFreeSpaces, vv) } else { - glog.Fatalf("The value specified in -minFreeSpacePercent not a valid value %s", freeString) + glog.Fatalf("The value specified in -minFreeSpace not a valid value %s", freeString) } } - if len(v.minFreeSpacePercents) == 1 && len(v.folders) > 1 { + if len(v.minFreeSpaces) == 1 && len(v.folders) > 1 { for i := 0; i < len(v.folders)-1; i++ { - v.minFreeSpacePercents = append(v.minFreeSpacePercents, v.minFreeSpacePercents[0]) + v.minFreeSpaces = append(v.minFreeSpaces, v.minFreeSpaces[0]) } } - if len(v.folders) != len(v.minFreeSpacePercents) { - glog.Fatalf("%d directories by -dir, but only %d minFreeSpacePercent is set by -minFreeSpacePercent", len(v.folders), len(v.minFreeSpacePercents)) + if len(v.folders) != len(v.minFreeSpaces) { + glog.Fatalf("%d directories by -dir, but only %d minFreeSpacePercent is set by -minFreeSpacePercent", len(v.folders), len(v.minFreeSpaces)) } // set disk types @@ -231,7 +233,7 @@ func (v VolumeServerOptions) startVolumeServer(volumeFolders, maxVolumeCounts, v volumeServer := weed_server.NewVolumeServer(volumeMux, publicVolumeMux, *v.ip, *v.port, *v.publicUrl, - v.folders, v.folderMaxLimits, v.minFreeSpacePercents, diskTypes, + v.folders, v.folderMaxLimits, v.minFreeSpaces, diskTypes, *v.idxFolder, volumeNeedleMapKind, strings.Split(masters, ","), 5, *v.dataCenter, *v.rack, diff --git a/weed/server/volume_server.go b/weed/server/volume_server.go index e11d607a4..f29f12148 100644 --- a/weed/server/volume_server.go +++ b/weed/server/volume_server.go @@ -43,7 +43,7 @@ type VolumeServer struct { func NewVolumeServer(adminMux, publicMux *http.ServeMux, ip string, port int, publicUrl string, - folders []string, maxCounts []int, minFreeSpacePercents []float32, diskTypes []types.DiskType, + folders []string, maxCounts []int, minFreeSpaces []float32, diskTypes []types.DiskType, idxFolder string, needleMapKind storage.NeedleMapKind, masterNodes []string, pulseSeconds int, @@ -85,7 +85,7 @@ func NewVolumeServer(adminMux, publicMux *http.ServeMux, ip string, vs.checkWithMaster() - vs.store = storage.NewStore(vs.grpcDialOption, port, ip, publicUrl, folders, maxCounts, minFreeSpacePercents, idxFolder, vs.needleMapKind, diskTypes) + vs.store = storage.NewStore(vs.grpcDialOption, port, ip, publicUrl, folders, maxCounts, minFreeSpaces, idxFolder, vs.needleMapKind, diskTypes) vs.guard = security.NewGuard(whiteList, signingKey, expiresAfterSec, readSigningKey, readExpiresAfterSec) handleStaticResources(adminMux) diff --git a/weed/storage/disk_location.go b/weed/storage/disk_location.go index ed4e00312..3246718c8 100644 --- a/weed/storage/disk_location.go +++ b/weed/storage/disk_location.go @@ -23,9 +23,10 @@ type DiskLocation struct { DiskType types.DiskType MaxVolumeCount int OriginalMaxVolumeCount int - MinFreeSpacePercent float32 - volumes map[needle.VolumeId]*Volume - volumesLock sync.RWMutex + // MinFreeSpace limits the minimum free space (<=100 as percentage, > 100 as bytes) + MinFreeSpace float32 + volumes map[needle.VolumeId]*Volume + volumesLock sync.RWMutex // erasure coding ecVolumes map[needle.VolumeId]*erasure_coding.EcVolume @@ -34,7 +35,7 @@ type DiskLocation struct { isDiskSpaceLow bool } -func NewDiskLocation(dir string, maxVolumeCount int, minFreeSpacePercent float32, idxDir string, diskType types.DiskType) *DiskLocation { +func NewDiskLocation(dir string, maxVolumeCount int, minFreeSpace float32, idxDir string, diskType types.DiskType) *DiskLocation { dir = util.ResolvePath(dir) if idxDir == "" { idxDir = dir @@ -47,7 +48,7 @@ func NewDiskLocation(dir string, maxVolumeCount int, minFreeSpacePercent float32 DiskType: diskType, MaxVolumeCount: maxVolumeCount, OriginalMaxVolumeCount: maxVolumeCount, - MinFreeSpacePercent: minFreeSpacePercent, + MinFreeSpace: minFreeSpace, } location.volumes = make(map[needle.VolumeId]*Volume) location.ecVolumes = make(map[needle.VolumeId]*erasure_coding.EcVolume) @@ -361,14 +362,19 @@ func (l *DiskLocation) CheckDiskSpace() { stats.VolumeServerResourceGauge.WithLabelValues(l.Directory, "all").Set(float64(s.All)) stats.VolumeServerResourceGauge.WithLabelValues(l.Directory, "used").Set(float64(s.Used)) stats.VolumeServerResourceGauge.WithLabelValues(l.Directory, "free").Set(float64(s.Free)) - if (s.PercentFree < l.MinFreeSpacePercent) != l.isDiskSpaceLow { + + isLow := l.MinFreeSpace < 100 && s.PercentFree < l.MinFreeSpace || s.Free < uint64(l.MinFreeSpace) + if isLow != l.isDiskSpaceLow { l.isDiskSpaceLow = !l.isDiskSpaceLow } + + logLevel := glog.Level(4) if l.isDiskSpaceLow { - glog.V(0).Infof("dir %s freePercent %.2f%% < min %.2f%%, isLowDiskSpace: %v", dir, s.PercentFree, l.MinFreeSpacePercent, l.isDiskSpaceLow) - } else { - glog.V(4).Infof("dir %s freePercent %.2f%% < min %.2f%%, isLowDiskSpace: %v", dir, s.PercentFree, l.MinFreeSpacePercent, l.isDiskSpaceLow) + logLevel = glog.Level(0) } + + glog.V(logLevel).Infof("dir %s freePercent %.2f%% < min %.2f%%, isLowDiskSpace: %v", + dir, s.PercentFree, l.MinFreeSpace, l.isDiskSpaceLow) } time.Sleep(time.Minute) } diff --git a/weed/storage/store.go b/weed/storage/store.go index 6be15a4c9..18496890c 100644 --- a/weed/storage/store.go +++ b/weed/storage/store.go @@ -52,11 +52,11 @@ func (s *Store) String() (str string) { return } -func NewStore(grpcDialOption grpc.DialOption, port int, ip, publicUrl string, dirnames []string, maxVolumeCounts []int, minFreeSpacePercents []float32, idxFolder string, needleMapKind NeedleMapKind, diskTypes []DiskType) (s *Store) { +func NewStore(grpcDialOption grpc.DialOption, port int, ip, publicUrl string, dirnames []string, maxVolumeCounts []int, minFreeSpaces []float32, idxFolder string, needleMapKind NeedleMapKind, diskTypes []DiskType) (s *Store) { s = &Store{grpcDialOption: grpcDialOption, Port: port, Ip: ip, PublicUrl: publicUrl, NeedleMapKind: needleMapKind} s.Locations = make([]*DiskLocation, 0) for i := 0; i < len(dirnames); i++ { - location := NewDiskLocation(dirnames[i], maxVolumeCounts[i], minFreeSpacePercents[i], idxFolder, diskTypes[i]) + location := NewDiskLocation(dirnames[i], maxVolumeCounts[i], minFreeSpaces[i], idxFolder, diskTypes[i]) location.loadExistingVolumes(needleMapKind) s.Locations = append(s.Locations, location) stats.VolumeServerMaxVolumeCounter.Add(float64(maxVolumeCounts[i])) diff --git a/weed/util/bytes.go b/weed/util/bytes.go index c2a4df108..260e5067e 100644 --- a/weed/util/bytes.go +++ b/weed/util/bytes.go @@ -5,8 +5,13 @@ import ( "crypto/md5" "crypto/rand" "encoding/base64" + "errors" "fmt" "io" + "math" + "strconv" + "strings" + "unicode" ) // BytesToHumanReadable returns the converted human readable representation of the bytes. @@ -161,3 +166,105 @@ func NewBytesReader(b []byte) *BytesReader { Reader: bytes.NewReader(b), } } + +// EmptyTo returns to if s is empty. +func EmptyTo(s, to string) string { + if s == "" { + return to + } + + return s +} + +var ErrMinFreeSpaceBadValue = errors.New("minFreeSpace is invalid") + +// ParseMinFreeSpace parses min free space expression s as percentage like 1,10 or human readable size like 10G +func ParseMinFreeSpace(s string) (float32, error) { + if value, e := strconv.ParseFloat(s, 32); e == nil { + if value < 0 || value > 100 { + return 0, ErrMinFreeSpaceBadValue + } + return float32(value), nil + } else if directSize, e2 := ParseBytes(s); e2 == nil { + if directSize <= 100 { + return 0, ErrMinFreeSpaceBadValue + } + return float32(directSize), nil + } + + return 0, ErrMinFreeSpaceBadValue +} + +// ParseBytes parses a string representation of bytes into the number +// of bytes it represents. +// +// See Also: Bytes, IBytes. +// +// ParseBytes("42MB") -> 42000000, nil +// ParseBytes("42 MB") -> 42000000, nil +// ParseBytes("42 mib") -> 44040192, nil +func ParseBytes(s string) (uint64, error) { + lastDigit := 0 + hasComma := false + for _, r := range s { + if !(unicode.IsDigit(r) || r == '.' || r == ',') { + break + } + if r == ',' { + hasComma = true + } + lastDigit++ + } + + num := s[:lastDigit] + if hasComma { + num = strings.Replace(num, ",", "", -1) + } + + f, err := strconv.ParseFloat(num, 64) + if err != nil { + return 0, err + } + + extra := strings.ToLower(strings.TrimSpace(s[lastDigit:])) + if m, ok := bytesSizeTable[extra]; ok { + f *= float64(m) + if f >= math.MaxUint64 { + return 0, fmt.Errorf("too large: %v", s) + } + return uint64(f), nil + } + + return 0, fmt.Errorf("unhandled size name: %v", extra) +} + +var bytesSizeTable = map[string]uint64{ + "b": Byte, "kib": KiByte, "kb": KByte, "mib": MiByte, "mb": MByte, "gib": GiByte, "gb": GByte, + "tib": TiByte, "tb": TByte, "pib": PiByte, "pb": PByte, "eib": EiByte, "eb": EByte, + // Without suffix + "": Byte, "ki": KiByte, "k": KByte, "mi": MiByte, "m": MByte, "gi": GiByte, "g": GByte, + "ti": TiByte, "t": TByte, "pi": PiByte, "p": PByte, "ei": EiByte, "e": EByte, +} + +// IEC Sizes. +// kibis of bits +const ( + Byte = 1 << (iota * 10) + KiByte + MiByte + GiByte + TiByte + PiByte + EiByte +) + +// SI Sizes. +const ( + IByte = 1 + KByte = IByte * 1000 + MByte = KByte * 1000 + GByte = MByte * 1000 + TByte = GByte * 1000 + PByte = TByte * 1000 + EByte = PByte * 1000 +) diff --git a/weed/util/bytes_test.go b/weed/util/bytes_test.go new file mode 100644 index 000000000..4a9c25e52 --- /dev/null +++ b/weed/util/bytes_test.go @@ -0,0 +1,85 @@ +package util + +import "testing" + +func TestParseMinFreeSpace(t *testing.T) { + tests := []struct { + in string + ok bool + value float32 + }{ + {in: "42", ok: true, value: 42}, + {in: "-1", ok: false, value: 0}, + {in: "101", ok: false, value: 0}, + {in: "100B", ok: false, value: 0}, + {in: "100Ki", ok: true, value: 100 * 1024}, + {in: "100GiB", ok: true, value: 100 * 1024 * 1024 * 1024}, + {in: "42M", ok: true, value: 42 * 1000 * 1000}, + } + + for _, p := range tests { + got, err := ParseMinFreeSpace(p.in) + if p.ok != (err == nil) { + t.Errorf("failed to test %v", p.in) + } + if p.ok && err == nil && got != p.value { + t.Errorf("failed to test %v", p.in) + } + } +} + +func TestByteParsing(t *testing.T) { + tests := []struct { + in string + exp uint64 + }{ + {"42", 42}, + {"42MB", 42000000}, + {"42MiB", 44040192}, + {"42mb", 42000000}, + {"42mib", 44040192}, + {"42MIB", 44040192}, + {"42 MB", 42000000}, + {"42 MiB", 44040192}, + {"42 mb", 42000000}, + {"42 mib", 44040192}, + {"42 MIB", 44040192}, + {"42.5MB", 42500000}, + {"42.5MiB", 44564480}, + {"42.5 MB", 42500000}, + {"42.5 MiB", 44564480}, + // No need to say B + {"42M", 42000000}, + {"42Mi", 44040192}, + {"42m", 42000000}, + {"42mi", 44040192}, + {"42MI", 44040192}, + {"42 M", 42000000}, + {"42 Mi", 44040192}, + {"42 m", 42000000}, + {"42 mi", 44040192}, + {"42 MI", 44040192}, + {"42.5M", 42500000}, + {"42.5Mi", 44564480}, + {"42.5 M", 42500000}, + {"42.5 Mi", 44564480}, + // Bug #42 + {"1,005.03 MB", 1005030000}, + // Large testing, breaks when too much larger than + // this. + {"12.5 EB", uint64(12.5 * float64(EByte))}, + {"12.5 E", uint64(12.5 * float64(EByte))}, + {"12.5 EiB", uint64(12.5 * float64(EiByte))}, + } + + for _, p := range tests { + got, err := ParseBytes(p.in) + if err != nil { + t.Errorf("Couldn't parse %v: %v", p.in, err) + } + if got != p.exp { + t.Errorf("Expected %v for %v, got %v", + p.exp, p.in, got) + } + } +}