From 308a48c0c20627ea47ddbfe9ba092d1235ab965b Mon Sep 17 00:00:00 2001 From: guol-fnst Date: Wed, 13 Jul 2022 09:40:46 +0800 Subject: [PATCH 1/2] optimiz concurrency user can customize number of workers via env "GOMAXPROCS" --- weed/storage/disk_location.go | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/weed/storage/disk_location.go b/weed/storage/disk_location.go index a2a63acbb..69d249259 100644 --- a/weed/storage/disk_location.go +++ b/weed/storage/disk_location.go @@ -5,6 +5,7 @@ import ( "os" "path/filepath" "runtime" + "strconv" "strings" "sync" "time" @@ -208,8 +209,14 @@ func (l *DiskLocation) concurrentLoadingVolumes(needleMapKind NeedleMapKind, con func (l *DiskLocation) loadExistingVolumes(needleMapKind NeedleMapKind) { workerNum := runtime.NumCPU() - if workerNum <= 10 { - workerNum = 10 + val, ok := os.LookupEnv("GOMAXPROCS") + if ok { + num, err := strconv.Atoi(val) + if err != nil || num < 1 { + num = 10 + glog.Warningf("failed to set worker number from GOMAXPROCS , set to default:10") + } + workerNum = num } l.concurrentLoadingVolumes(needleMapKind, workerNum) glog.V(0).Infof("Store started on dir: %s with %d volumes max %d", l.Directory, len(l.volumes), l.MaxVolumeCount) From 300b383cdf7ab607c92417256dfdc5ca52ea24b0 Mon Sep 17 00:00:00 2001 From: guol-fnst Date: Thu, 14 Jul 2022 10:37:13 +0800 Subject: [PATCH 2/2] use 10 or numCPU workers if env is not found --- weed/storage/disk_location.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/weed/storage/disk_location.go b/weed/storage/disk_location.go index 69d249259..8af8ea663 100644 --- a/weed/storage/disk_location.go +++ b/weed/storage/disk_location.go @@ -217,6 +217,10 @@ func (l *DiskLocation) loadExistingVolumes(needleMapKind NeedleMapKind) { glog.Warningf("failed to set worker number from GOMAXPROCS , set to default:10") } workerNum = num + } else { + if workerNum <= 10 { + workerNum = 10 + } } l.concurrentLoadingVolumes(needleMapKind, workerNum) glog.V(0).Infof("Store started on dir: %s with %d volumes max %d", l.Directory, len(l.volumes), l.MaxVolumeCount)