From 44e899464f47862f7a346bc67e3f24e8e6807f89 Mon Sep 17 00:00:00 2001 From: bingoohuang Date: Mon, 21 Jan 2019 15:53:11 +0800 Subject: [PATCH] volume server options add diskWaterMark (default 10g) --- weed/command/server.go | 2 +- weed/command/volume.go | 21 ++++++++++----------- weed/server/volume_server.go | 5 +++-- weed/storage/needle_map.go | 13 +++++++++++++ weed/storage/store.go | 5 +++-- 5 files changed, 30 insertions(+), 16 deletions(-) diff --git a/weed/command/server.go b/weed/command/server.go index ba5305a97..02360c1cd 100644 --- a/weed/command/server.go +++ b/weed/command/server.go @@ -92,7 +92,7 @@ func init() { serverOptions.v.fixJpgOrientation = cmdServer.Flag.Bool("volume.images.fix.orientation", false, "Adjust jpg orientation when uploading.") serverOptions.v.readRedirect = cmdServer.Flag.Bool("volume.read.redirect", true, "Redirect moved or non-local volumes.") serverOptions.v.publicUrl = cmdServer.Flag.String("volume.publicUrl", "", "publicly accessible address") - + serverOptions.v.diskWaterMark = cmdVolume.Flag.String("volume.diskWaterMark", "10GB", "disk watermark low to switch volume to read-only(eg 5G,100M,5GiB,100MiB)") } func runServer(cmd *Command, args []string) bool { diff --git a/weed/command/volume.go b/weed/command/volume.go index f7f7c9eb1..96e013c02 100644 --- a/weed/command/volume.go +++ b/weed/command/volume.go @@ -14,6 +14,7 @@ import ( "github.com/chrislusf/seaweedfs/weed/server" "github.com/chrislusf/seaweedfs/weed/storage" "github.com/chrislusf/seaweedfs/weed/util" + "github.com/dustin/go-humanize" "google.golang.org/grpc/reflection" ) @@ -41,6 +42,7 @@ type VolumeServerOptions struct { readRedirect *bool cpuProfile *string memProfile *string + diskWaterMark *string } func init() { @@ -61,6 +63,7 @@ func init() { v.readRedirect = cmdVolume.Flag.Bool("read.redirect", true, "Redirect moved or non-local volumes.") v.cpuProfile = cmdVolume.Flag.String("cpuprofile", "", "cpu profile output file") v.memProfile = cmdVolume.Flag.String("memprofile", "", "memory profile output file") + v.diskWaterMark = cmdVolume.Flag.String("diskWaterMark", "10g", "disk watermark low to switch volume to read-only(eg 5G,100M,5GiB,100MiB)") } var cmdVolume = &Command{ @@ -133,26 +136,22 @@ func (v VolumeServerOptions) startVolumeServer(volumeFolders, maxVolumeCounts, v publicVolumeMux = http.NewServeMux() } - volumeNeedleMapKind := storage.NeedleMapInMemory - switch *v.indexType { - case "leveldb": - volumeNeedleMapKind = storage.NeedleMapLevelDb - case "boltdb": - volumeNeedleMapKind = storage.NeedleMapBoltDb - case "btree": - volumeNeedleMapKind = storage.NeedleMapBtree - } + volumeNeedleMapKind := storage.ParseVolumeNeedleMapKind(*v.indexType) masters := *v.masters + diskWaterMarkBytes, err := humanize.ParseBytes(*v.diskWaterMark) + if err != nil { + glog.Fatalf("Check diskWaterMark %s error:%v ", *v.diskWaterMark, err) + } + volumeServer := weed_server.NewVolumeServer(volumeMux, publicVolumeMux, *v.ip, *v.port, *v.publicUrl, v.folders, v.folderMaxLimits, volumeNeedleMapKind, strings.Split(masters, ","), *v.pulseSeconds, *v.dataCenter, *v.rack, v.whiteList, - *v.fixJpgOrientation, *v.readRedirect, - ) + *v.fixJpgOrientation, *v.readRedirect, diskWaterMarkBytes) listeningAddress := *v.bindIp + ":" + strconv.Itoa(*v.port) glog.V(0).Infof("Start Seaweed volume server %s at %s", util.VERSION, listeningAddress) diff --git a/weed/server/volume_server.go b/weed/server/volume_server.go index 0914e81b0..8ed7af2ed 100644 --- a/weed/server/volume_server.go +++ b/weed/server/volume_server.go @@ -30,7 +30,8 @@ func NewVolumeServer(adminMux, publicMux *http.ServeMux, ip string, dataCenter string, rack string, whiteList []string, fixJpgOrientation bool, - readRedirect bool) *VolumeServer { + readRedirect bool, + diskWaterMark uint64) *VolumeServer { vs := &VolumeServer{ pulseSeconds: pulseSeconds, dataCenter: dataCenter, @@ -40,7 +41,7 @@ func NewVolumeServer(adminMux, publicMux *http.ServeMux, ip string, ReadRedirect: readRedirect, } vs.MasterNodes = masterNodes - vs.store = storage.NewStore(port, ip, publicUrl, folders, maxCounts, vs.needleMapKind) + vs.store = storage.NewStore(port, ip, publicUrl, folders, maxCounts, vs.needleMapKind, diskWaterMark) vs.guard = security.NewGuard(whiteList, "") diff --git a/weed/storage/needle_map.go b/weed/storage/needle_map.go index 6d815679b..a143f199c 100644 --- a/weed/storage/needle_map.go +++ b/weed/storage/needle_map.go @@ -20,6 +20,19 @@ const ( NeedleMapBtree ) +func ParseVolumeNeedleMapKind(indexType string) NeedleMapType { + switch indexType { + case "leveldb": + return NeedleMapLevelDb + case "boltdb": + return NeedleMapBoltDb + case "btree": + return NeedleMapBtree + default: + return NeedleMapInMemory + } +} + type NeedleMapper interface { Put(key NeedleId, offset Offset, size uint32) error Get(key NeedleId) (element *needle.NeedleValue, ok bool) diff --git a/weed/storage/store.go b/weed/storage/store.go index 3c00edb95..4e48d48dd 100644 --- a/weed/storage/store.go +++ b/weed/storage/store.go @@ -37,8 +37,9 @@ func (s *Store) String() (str string) { return } -func NewStore(port int, ip, publicUrl string, dirNames []string, maxVolumeCounts []int, needleMapKind NeedleMapType) (s *Store) { - s = &Store{Port: port, Ip: ip, PublicUrl: publicUrl, NeedleMapType: needleMapKind} +func NewStore(port int, ip, publicUrl string, dirNames []string, maxVolumeCounts []int, needleMapKind NeedleMapType, + diskWaterMark uint64) (s *Store) { + s = &Store{Port: port, Ip: ip, PublicUrl: publicUrl, NeedleMapType: needleMapKind, DiskWatermark: diskWaterMark} s.Locations = make([]*DiskLocation, 0) for i := 0; i < len(dirNames); i++ { location := NewDiskLocation(dirNames[i], maxVolumeCounts[i])