From 05b5f12f2e13d13e5ea3b983defead47f8b16a16 Mon Sep 17 00:00:00 2001 From: Konstantin Lebedev Date: Wed, 21 Oct 2020 20:48:51 +0500 Subject: [PATCH 01/11] multiplate DomainNames through comma --- weed/s3api/s3api_server.go | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/weed/s3api/s3api_server.go b/weed/s3api/s3api_server.go index 1ab80c3ee..b1e1cfe80 100644 --- a/weed/s3api/s3api_server.go +++ b/weed/s3api/s3api_server.go @@ -3,6 +3,7 @@ package s3api import ( "fmt" "net/http" + "strings" "github.com/gorilla/mux" "google.golang.org/grpc" @@ -39,10 +40,13 @@ func (s3a *S3ApiServer) registerRouter(router *mux.Router) { apiRouter := router.PathPrefix("/").Subrouter() var routers []*mux.Router if s3a.option.DomainName != "" { - routers = append(routers, apiRouter.Host( - fmt.Sprintf("%s.%s:%d", "{bucket:.+}", s3a.option.DomainName, s3a.option.Port)).Subrouter()) - routers = append(routers, apiRouter.Host( - fmt.Sprintf("%s.%s", "{bucket:.+}", s3a.option.DomainName)).Subrouter()) + domainNames := strings.Split(s3a.option.DomainName, ",") + for _, domainName := range domainNames { + routers = append(routers, apiRouter.Host( + fmt.Sprintf("%s.%s:%d", "{bucket:.+}", domainName, s3a.option.Port)).Subrouter()) + routers = append(routers, apiRouter.Host( + fmt.Sprintf("%s.%s", "{bucket:.+}", domainName)).Subrouter()) + } } routers = append(routers, apiRouter.PathPrefix("/{bucket}").Subrouter()) From bd103c143a8db12687d2d60829aac6bb6d42d74c Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Wed, 21 Oct 2020 19:28:59 -0700 Subject: [PATCH 02/11] add lock for vidCache --- weed/filer/reader_at.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/weed/filer/reader_at.go b/weed/filer/reader_at.go index f73f72429..04c64d449 100644 --- a/weed/filer/reader_at.go +++ b/weed/filer/reader_at.go @@ -38,9 +38,12 @@ type LookupFileIdFunctionType func(fileId string) (targetUrls []string, err erro func LookupFn(filerClient filer_pb.FilerClient) LookupFileIdFunctionType { vidCache := make(map[string]*filer_pb.Locations) + var vicCacheLock sync.RWMutex return func(fileId string) (targetUrls []string, err error) { vid := VolumeId(fileId) + vicCacheLock.RLock() locations, found := vidCache[vid] + vicCacheLock.RUnlock() waitTime := time.Second for !found && waitTime < ReadWaitTime { @@ -58,7 +61,9 @@ func LookupFn(filerClient filer_pb.FilerClient) LookupFileIdFunctionType { glog.V(0).Infof("failed to locate %s", fileId) return fmt.Errorf("failed to locate %s", fileId) } + vicCacheLock.Lock() vidCache[vid] = locations + vicCacheLock.Unlock() return nil }) From 9884dfc3692dd83d744644cca65fbf5f7b95b126 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Wed, 21 Oct 2020 19:29:51 -0700 Subject: [PATCH 03/11] use bytebuffer for interval list --- go.mod | 2 +- go.sum | 11 ++--- weed/filesys/dirty_page.go | 4 +- weed/filesys/dirty_page_interval.go | 72 +++++++++++++++++++++-------- 4 files changed, 61 insertions(+), 28 deletions(-) diff --git a/go.mod b/go.mod index 2a65d47a1..dd3d7841a 100644 --- a/go.mod +++ b/go.mod @@ -34,7 +34,6 @@ require ( github.com/gorilla/mux v1.7.4 github.com/gorilla/websocket v1.4.1 // indirect github.com/grpc-ecosystem/grpc-gateway v1.11.0 // indirect - github.com/hashicorp/golang-lru v0.5.3 // indirect github.com/jcmturner/gofork v1.0.0 // indirect github.com/json-iterator/go v1.1.10 github.com/karlseguin/ccache v2.0.3+incompatible @@ -70,6 +69,7 @@ require ( github.com/syndtr/goleveldb v1.0.0 github.com/tidwall/gjson v1.3.2 github.com/tidwall/match v1.0.1 + github.com/valyala/bytebufferpool v1.0.0 github.com/willf/bitset v1.1.10 // indirect github.com/willf/bloom v2.0.3+incompatible github.com/wsxiaoys/terminal v0.0.0-20160513160801-0940f3fc43a0 // indirect diff --git a/go.sum b/go.sum index ce8ff4a8c..b2f9b929e 100644 --- a/go.sum +++ b/go.sum @@ -45,8 +45,6 @@ github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5 github.com/aws/aws-sdk-go v1.15.27/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZoCYDt7FT0= github.com/aws/aws-sdk-go v1.19.18/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go v1.19.45/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= -github.com/aws/aws-sdk-go v1.23.13 h1:l/NG+mgQFRGG3dsFzEj0jw9JIs/zYdtU6MXhY1WIDmM= -github.com/aws/aws-sdk-go v1.23.13/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go v1.33.5 h1:p2fr1ryvNTU6avUWLI+/H7FGv0TBIjzVM5WDgXBBv4U= github.com/aws/aws-sdk-go v1.33.5/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= @@ -67,8 +65,6 @@ github.com/census-instrumentation/opencensus-proto v0.2.1 h1:glEXhBS5PSLLv4IXzLA github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= -github.com/chrislusf/raft v1.0.1 h1:Wa4ffkmkysW7cX3T/gMC/Mk3PhnOXhsqOVwQJcMndhw= -github.com/chrislusf/raft v1.0.1/go.mod h1:Ep5DP+mJSosjfKiix1uU7Lc2Df/SX4oGJEpZlXH5l68= github.com/chrislusf/raft v1.0.2-0.20201002174524-b13c3bfdb011 h1:vN1GvfLgDg8kIPCdhuVKAjlYpxG1B86jiKejB6MC/Q0= github.com/chrislusf/raft v1.0.2-0.20201002174524-b13c3bfdb011/go.mod h1:Ep5DP+mJSosjfKiix1uU7Lc2Df/SX4oGJEpZlXH5l68= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= @@ -127,6 +123,7 @@ github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4 h1:7HZCaLC5+BZpm github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4/go.mod h1:5tD+neXqOorC30/tWg0LCSkrqj/AR6gu8yY8/fpw1q0= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fortytw2/leaktest v1.2.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= +github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= github.com/frankban/quicktest v1.7.2 h1:2QxQoC1TS09S7fhCPsrvqYdvP1H5M1P1ih5ABm3BTYk= github.com/frankban/quicktest v1.7.2/go.mod h1:jaStnuzAqU1AJdCO0l53JDCJrVDKcS03DbaAcR7Ks/o= @@ -267,8 +264,6 @@ github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/b github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.3 h1:YPkqC67at8FYaadspW/6uE0COsBxS2656RLEr8Bppgk= -github.com/hashicorp/golang-lru v0.5.3/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= @@ -508,6 +503,8 @@ github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1 github.com/ugorji/go v1.1.4 h1:j4s+tAvLfL3bZyefP2SEWmhBzmuIlH/eqNuPdFPgngw= github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= +github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= +github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/willf/bitset v1.1.10 h1:NotGKqX0KwQ72NUzqrjZq5ipPNDQex9lo3WpaS8L2sc= github.com/willf/bitset v1.1.10/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= github.com/willf/bloom v2.0.3+incompatible h1:QDacWdqcAUI1MPOwIQZRy9kOR7yxfyEmxX8Wdm2/JPA= @@ -601,8 +598,6 @@ golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190619014844-b5b0513f8c1b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190909003024-a7b16738d86b h1:XfVGCX+0T4WOStkaOsJRllbsiImhB2jgVBGc9L0lPGc= -golang.org/x/net v0.0.0-20190909003024-a7b16738d86b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200202094626-16171245cfb2 h1:CCH4IOTTfewWjGOlSp+zGcjutRKlBEZQ6wTn8ozI/nI= golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= diff --git a/weed/filesys/dirty_page.go b/weed/filesys/dirty_page.go index 7a3e859f5..6fda134aa 100644 --- a/weed/filesys/dirty_page.go +++ b/weed/filesys/dirty_page.go @@ -54,7 +54,7 @@ func (pages *ContinuousDirtyPages) AddPage(offset int64, data []byte) { pages.intervals.AddInterval(data, offset) - if pages.intervals.TotalSize() > pages.f.wfs.option.ChunkSizeLimit { + if pages.intervals.TotalSize() >= pages.f.wfs.option.ChunkSizeLimit { pages.saveExistingLargestPageToStorage() } @@ -93,6 +93,8 @@ func (pages *ContinuousDirtyPages) saveExistingLargestPageToStorage() (hasSavedD pages.saveToStorage(maxList.ToReader(), maxList.Offset(), chunkSize) + maxList.Destroy() + return true } diff --git a/weed/filesys/dirty_page_interval.go b/weed/filesys/dirty_page_interval.go index f143fe3e4..ee23a7b23 100644 --- a/weed/filesys/dirty_page_interval.go +++ b/weed/filesys/dirty_page_interval.go @@ -5,6 +5,7 @@ import ( "io" "github.com/chrislusf/seaweedfs/weed/util" + "github.com/valyala/bytebufferpool" ) type IntervalNode struct { @@ -12,6 +13,15 @@ type IntervalNode struct { Offset int64 Size int64 Next *IntervalNode + Buffer *bytebufferpool.ByteBuffer +} + +func (l *IntervalNode) Bytes() []byte { + data := l.Data + if data == nil { + data = l.Buffer.Bytes() + } + return data } type IntervalLinkedList struct { @@ -23,16 +33,39 @@ type ContinuousIntervals struct { lists []*IntervalLinkedList } +func NewIntervalLinkedList(head, tail *IntervalNode) *IntervalLinkedList { + list := &IntervalLinkedList{ + Head: head, + Tail: tail, + } + return list +} + +func (list *IntervalLinkedList) Destroy() { + for t := list.Head; t != nil; t = t.Next { + if t.Buffer != nil { + bytebufferpool.Put(t.Buffer) + } + } +} + func (list *IntervalLinkedList) Offset() int64 { return list.Head.Offset } func (list *IntervalLinkedList) Size() int64 { return list.Tail.Offset + list.Tail.Size - list.Head.Offset } + func (list *IntervalLinkedList) addNodeToTail(node *IntervalNode) { // glog.V(4).Infof("add to tail [%d,%d) + [%d,%d) => [%d,%d)", list.Head.Offset, list.Tail.Offset+list.Tail.Size, node.Offset, node.Offset+node.Size, list.Head.Offset, node.Offset+node.Size) - list.Tail.Next = node - list.Tail = node + if list.Tail.Buffer == nil { + list.Tail.Buffer = bytebufferpool.Get() + list.Tail.Buffer.Write(list.Tail.Data) + list.Tail.Data = nil + } + list.Tail.Buffer.Write(node.Data) + list.Tail.Size += int64(len(node.Data)) + return } func (list *IntervalLinkedList) addNodeToHead(node *IntervalNode) { // glog.V(4).Infof("add to head [%d,%d) + [%d,%d) => [%d,%d)", node.Offset, node.Offset+node.Size, list.Head.Offset, list.Tail.Offset+list.Tail.Size, node.Offset, list.Tail.Offset+list.Tail.Size) @@ -47,7 +80,7 @@ func (list *IntervalLinkedList) ReadData(buf []byte, start, stop int64) { nodeStart, nodeStop := max(start, t.Offset), min(stop, t.Offset+t.Size) if nodeStart < nodeStop { // glog.V(0).Infof("copying start=%d stop=%d t=[%d,%d) t.data=%d => bufSize=%d nodeStart=%d, nodeStop=%d", start, stop, t.Offset, t.Offset+t.Size, len(t.Data), len(buf), nodeStart, nodeStop) - copy(buf[nodeStart-start:], t.Data[nodeStart-t.Offset:nodeStop-t.Offset]) + copy(buf[nodeStart-start:], t.Bytes()[nodeStart-t.Offset:nodeStop-t.Offset]) } if t.Next == nil { @@ -72,8 +105,15 @@ func subList(list *IntervalLinkedList, start, stop int64) *IntervalLinkedList { // skip non overlapping IntervalNode continue } + data := t.Bytes()[nodeStart-t.Offset : nodeStop-t.Offset] + if t.Data == nil { + // need to clone if the bytes is from byte buffer + t := make([]byte, len(data)) + copy(t, data) + data = t + } nodes = append(nodes, &IntervalNode{ - Data: t.Data[nodeStart-t.Offset : nodeStop-t.Offset], + Data: data, Offset: nodeStart, Size: nodeStop - nodeStart, Next: nil, @@ -82,10 +122,7 @@ func subList(list *IntervalLinkedList, start, stop int64) *IntervalLinkedList { for i := 1; i < len(nodes); i++ { nodes[i-1].Next = nodes[i] } - return &IntervalLinkedList{ - Head: nodes[0], - Tail: nodes[len(nodes)-1], - } + return NewIntervalLinkedList(nodes[0], nodes[len(nodes)-1]) } func (c *ContinuousIntervals) AddInterval(data []byte, offset int64) { @@ -144,10 +181,7 @@ func (c *ContinuousIntervals) AddInterval(data []byte, offset int64) { nextList.addNodeToHead(interval) } if prevList == nil && nextList == nil { - c.lists = append(c.lists, &IntervalLinkedList{ - Head: interval, - Tail: interval, - }) + c.lists = append(c.lists, NewIntervalLinkedList(interval, interval)) } return @@ -155,11 +189,13 @@ func (c *ContinuousIntervals) AddInterval(data []byte, offset int64) { func (c *ContinuousIntervals) RemoveLargestIntervalLinkedList() *IntervalLinkedList { var maxSize int64 - maxIndex := -1 + maxIndex, maxOffset := -1, int64(-1) + println("in memory list:", len(c.lists)) for k, list := range c.lists { - if maxSize <= list.Size() { - maxSize = list.Size() - maxIndex = k + listSize := list.Size() + if maxSize < listSize || (maxSize == listSize && list.Offset() < maxOffset ) { + maxSize = listSize + maxIndex, maxOffset = k, list.Offset() } } if maxSize <= 0 { @@ -202,10 +238,10 @@ func (c *ContinuousIntervals) ReadDataAt(data []byte, startOffset int64) (maxSto func (l *IntervalLinkedList) ToReader() io.Reader { var readers []io.Reader t := l.Head - readers = append(readers, util.NewBytesReader(t.Data)) + readers = append(readers, util.NewBytesReader(t.Bytes())) for t.Next != nil { t = t.Next - readers = append(readers, bytes.NewReader(t.Data)) + readers = append(readers, bytes.NewReader(t.Bytes())) } if len(readers) == 1 { return readers[0] From b5917bf846283f79fc38a4a9980fc4f5db10c4dd Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Wed, 21 Oct 2020 19:31:26 -0700 Subject: [PATCH 04/11] clean up --- weed/filesys/dirty_page_interval.go | 1 - 1 file changed, 1 deletion(-) diff --git a/weed/filesys/dirty_page_interval.go b/weed/filesys/dirty_page_interval.go index ee23a7b23..56590126f 100644 --- a/weed/filesys/dirty_page_interval.go +++ b/weed/filesys/dirty_page_interval.go @@ -190,7 +190,6 @@ func (c *ContinuousIntervals) AddInterval(data []byte, offset int64) { func (c *ContinuousIntervals) RemoveLargestIntervalLinkedList() *IntervalLinkedList { var maxSize int64 maxIndex, maxOffset := -1, int64(-1) - println("in memory list:", len(c.lists)) for k, list := range c.lists { listSize := list.Size() if maxSize < listSize || (maxSize == listSize && list.Offset() < maxOffset ) { From f5ee19e0dbf761a0adbaa84c337e3ca35e605baf Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Wed, 21 Oct 2020 20:44:01 -0700 Subject: [PATCH 05/11] shortcut for appending operations --- weed/filesys/dirty_page_interval.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/weed/filesys/dirty_page_interval.go b/weed/filesys/dirty_page_interval.go index 56590126f..f644bea0b 100644 --- a/weed/filesys/dirty_page_interval.go +++ b/weed/filesys/dirty_page_interval.go @@ -129,6 +129,15 @@ func (c *ContinuousIntervals) AddInterval(data []byte, offset int64) { interval := &IntervalNode{Data: data, Offset: offset, Size: int64(len(data))} + // append to the tail and return + if len(c.lists) == 1 { + lastSpan := c.lists[0] + if lastSpan.Tail.Offset + lastSpan.Tail.Size == offset { + lastSpan.addNodeToTail(interval) + return + } + } + var newLists []*IntervalLinkedList for _, list := range c.lists { // if list is to the left of new interval, add to the new list From 4ff2ceee33e24bab025eaa9ce547783e6cbab2db Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Wed, 21 Oct 2020 22:05:58 -0700 Subject: [PATCH 06/11] UI fix on rendering EC volumes addressing UI problem with https://github.com/chrislusf/seaweedfs/issues/1551 --- weed/storage/erasure_coding/ec_volume.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/weed/storage/erasure_coding/ec_volume.go b/weed/storage/erasure_coding/ec_volume.go index 785f33ec4..71fe884df 100644 --- a/weed/storage/erasure_coding/ec_volume.go +++ b/weed/storage/erasure_coding/ec_volume.go @@ -145,9 +145,9 @@ func (ev *EcVolume) FileName() string { } -func (ev *EcVolume) ShardSize() int64 { +func (ev *EcVolume) ShardSize() uint64 { if len(ev.Shards) > 0 { - return ev.Shards[0].Size() + return uint64(ev.Shards[0].Size()) } return 0 } From 2bc05cfbc896353082075f91aea977f7fe32f0eb Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Wed, 21 Oct 2020 22:26:41 -0700 Subject: [PATCH 07/11] fix typo --- weed/shell/command_ec_balance.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/weed/shell/command_ec_balance.go b/weed/shell/command_ec_balance.go index bb280b7d9..7117f52df 100644 --- a/weed/shell/command_ec_balance.go +++ b/weed/shell/command_ec_balance.go @@ -174,7 +174,7 @@ func balanceEcVolumes(commandEnv *CommandEnv, collection string, allEcNodes []*E } if err := balanceEcShardsWithinRacks(commandEnv, allEcNodes, racks, collection, applyBalancing); err != nil { - return fmt.Errorf("balance across racks collection %s ec shards: %v", collection, err) + return fmt.Errorf("balance within racks collection %s ec shards: %v", collection, err) } return nil From 720b1d9b88130b7faa466f734807c1c1237611d3 Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Wed, 21 Oct 2020 23:15:48 -0700 Subject: [PATCH 08/11] adding locking to avoid nil VolumeLocationList fix panic: runtime error: invalid memory address or nil pointer dereference Oct 22 00:53:44 bedb-master1 weed[8055]: [signal SIGSEGV: segmentation violation code=0x1 addr=0x8 pc=0x17658da] Oct 22 00:53:44 bedb-master1 weed[8055]: goroutine 310 [running]: Oct 22 00:53:44 bedb-master1 weed[8055]: github.com/chrislusf/seaweedfs/weed/topology.(*VolumeLocationList).Length(...) Oct 22 00:53:44 bedb-master1 weed[8055]: #011/root/seaweedfs/weed/topology/volume_location_list.go:35 Oct 22 00:53:44 bedb-master1 weed[8055]: github.com/chrislusf/seaweedfs/weed/topology.(*VolumeLayout).enoughCopies(...) Oct 22 00:53:44 bedb-master1 weed[8055]: #011/root/seaweedfs/weed/topology/volume_layout.go:376 Oct 22 00:53:44 bedb-master1 weed[8055]: github.com/chrislusf/seaweedfs/weed/topology.(*VolumeLayout).ensureCorrectWritables(0xc000111d50, 0xc000b55438) Oct 22 00:53:44 bedb-master1 weed[8055]: #011/root/seaweedfs/weed/topology/volume_layout.go:202 +0x5a Oct 22 00:53:44 bedb-master1 weed[8055]: github.com/chrislusf/seaweedfs/weed/topology.(*Topology).SyncDataNodeRegistration(0xc00042ac60, 0xc001454d30, 0x1, 0x1, 0xc0005fc000, 0xc00135de40, 0x4, 0xc00135de50, 0x10, 0x10d, ...) Oct 22 00:53:44 bedb-master1 weed[8055]: #011/root/seaweedfs/weed/topology/topology.go:224 +0x616 Oct 22 00:53:44 bedb-master1 weed[8055]: github.com/chrislusf/seaweedfs/weed/server.(*MasterServer).SendHeartbeat(0xc000162700, 0x23b97c0, 0xc000ae2c90, 0x0, 0x0) Oct 22 00:53:44 bedb-master1 weed[8055]: #011/root/seaweedfs/weed/server/master_grpc_server.go:106 +0x325 Oct 22 00:53:44 bedb-master1 weed[8055]: github.com/chrislusf/seaweedfs/weed/pb/master_pb._Seaweed_SendHeartbeat_Handler(0x1f8e7c0, 0xc000162700, 0x23b0a60, 0xc00024b440, 0x3172c38, 0xc000ab7100) Oct 22 00:53:44 bedb-master1 weed[8055]: #011/root/seaweedfs/weed/pb/master_pb/master.pb.go:4250 +0xad Oct 22 00:53:44 bedb-master1 weed[8055]: google.golang.org/grpc.(*Server).processStreamingRPC(0xc0001f31e0, 0x23bb800, 0xc000ac5500, 0xc000ab7100, 0xc0001fea80, 0x311fec0, 0x0, 0x0, 0x0) Oct 22 00:53:44 bedb-master1 weed[8055]: #011/root/go/pkg/mod/google.golang.org/grpc@v1.29.1/server.go:1329 +0xcd8 Oct 22 00:53:44 bedb-master1 weed[8055]: google.golang.org/grpc.(*Server).handleStream(0xc0001f31e0, 0x23bb800, 0xc000ac5500, 0xc000ab7100, 0x0) Oct 22 00:53:44 bedb-master1 weed[8055]: #011/root/go/pkg/mod/google.golang.org/grpc@v1.29.1/server.go:1409 +0xc5c Oct 22 00:53:44 bedb-master1 weed[8055]: google.golang.org/grpc.(*Server).serveStreams.func1.1(0xc0001ce8b0, 0xc0001f31e0, 0x23bb800, 0xc000ac5500, 0xc000ab7100) Oct 22 00:53:44 bedb-master1 weed[8055]: #011/root/go/pkg/mod/google.golang.org/grpc@v1.29.1/server.go:746 +0xa5 Oct 22 00:53:44 bedb-master1 weed[8055]: created by google.golang.org/grpc.(*Server).serveStreams.func1 Oct 22 00:53:44 bedb-master1 weed[8055]: #011/root/go/pkg/mod/google.golang.org/grpc@v1.29.1/server.go:744 +0xa5 Oct 22 00:53:44 bedb-master1 systemd[1]: weedmaster.service: Main process exited, code=exited, status=2/INVALIDARGUMENT Oct 22 00:53:44 bedb-master1 systemd[1]: weedmaster.service: Failed with result 'exit-code'. --- weed/topology/topology.go | 2 +- weed/topology/volume_layout.go | 7 +++++++ weed/topology/volume_location_list.go | 3 +++ 3 files changed, 11 insertions(+), 1 deletion(-) diff --git a/weed/topology/topology.go b/weed/topology/topology.go index a11a1bac6..e217617e9 100644 --- a/weed/topology/topology.go +++ b/weed/topology/topology.go @@ -221,7 +221,7 @@ func (t *Topology) SyncDataNodeRegistration(volumes []*master_pb.VolumeInformati } for _, v := range changedVolumes { vl := t.GetVolumeLayout(v.Collection, v.ReplicaPlacement, v.Ttl) - vl.ensureCorrectWritables(&v) + vl.EnsureCorrectWritables(&v) } return } diff --git a/weed/topology/volume_layout.go b/weed/topology/volume_layout.go index e7659e0eb..dddcfc9c9 100644 --- a/weed/topology/volume_layout.go +++ b/weed/topology/volume_layout.go @@ -198,6 +198,13 @@ func (vl *VolumeLayout) UnRegisterVolume(v *storage.VolumeInfo, dn *DataNode) { } } +func (vl *VolumeLayout) EnsureCorrectWritables(v *storage.VolumeInfo) { + vl.accessLock.Lock() + defer vl.accessLock.Unlock() + + vl.ensureCorrectWritables(v) +} + func (vl *VolumeLayout) ensureCorrectWritables(v *storage.VolumeInfo) { if vl.enoughCopies(v.Id) && vl.isWritable(v) { if !vl.oversizedVolumes.IsTrue(v.Id) { diff --git a/weed/topology/volume_location_list.go b/weed/topology/volume_location_list.go index 6acd70f2f..64c13ca52 100644 --- a/weed/topology/volume_location_list.go +++ b/weed/topology/volume_location_list.go @@ -32,6 +32,9 @@ func (dnll *VolumeLocationList) Head() *DataNode { } func (dnll *VolumeLocationList) Length() int { + if dnll == nil { + return 0 + } return len(dnll.list) } From a9a7005687abc7cdc4076d53c334c85281ae3188 Mon Sep 17 00:00:00 2001 From: Konstantin Lebedev Date: Thu, 22 Oct 2020 11:23:00 +0500 Subject: [PATCH 09/11] set desc of option --- weed/command/filer.go | 2 +- weed/command/s3.go | 2 +- weed/command/server.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/weed/command/filer.go b/weed/command/filer.go index 793401fe3..1ea334e61 100644 --- a/weed/command/filer.go +++ b/weed/command/filer.go @@ -69,7 +69,7 @@ func init() { // start s3 on filer filerStartS3 = cmdFiler.Flag.Bool("s3", false, "whether to start S3 gateway") filerS3Options.port = cmdFiler.Flag.Int("s3.port", 8333, "s3 server http listen port") - filerS3Options.domainName = cmdFiler.Flag.String("s3.domainName", "", "suffix of the host name, {bucket}.{domainName}") + filerS3Options.domainName = cmdFiler.Flag.String("s3.domainName", "", "suffix of the host name in comma separated list, {bucket}.{domainName}") filerS3Options.tlsPrivateKey = cmdFiler.Flag.String("s3.key.file", "", "path to the TLS private key file") filerS3Options.tlsCertificate = cmdFiler.Flag.String("s3.cert.file", "", "path to the TLS certificate file") filerS3Options.config = cmdFiler.Flag.String("s3.config", "", "path to the config file") diff --git a/weed/command/s3.go b/weed/command/s3.go index 7d0954c0c..ed5bb0b80 100644 --- a/weed/command/s3.go +++ b/weed/command/s3.go @@ -36,7 +36,7 @@ func init() { cmdS3.Run = runS3 // break init cycle s3StandaloneOptions.filer = cmdS3.Flag.String("filer", "localhost:8888", "filer server address") s3StandaloneOptions.port = cmdS3.Flag.Int("port", 8333, "s3 server http listen port") - s3StandaloneOptions.domainName = cmdS3.Flag.String("domainName", "", "suffix of the host name, {bucket}.{domainName}") + s3StandaloneOptions.domainName = cmdS3.Flag.String("domainName", "", "suffix of the host name in comma separated list, {bucket}.{domainName}") s3StandaloneOptions.config = cmdS3.Flag.String("config", "", "path to the config file") s3StandaloneOptions.tlsPrivateKey = cmdS3.Flag.String("key.file", "", "path to the TLS private key file") s3StandaloneOptions.tlsCertificate = cmdS3.Flag.String("cert.file", "", "path to the TLS certificate file") diff --git a/weed/command/server.go b/weed/command/server.go index c55d2cfee..6a78fb3f4 100644 --- a/weed/command/server.go +++ b/weed/command/server.go @@ -105,7 +105,7 @@ func init() { serverOptions.v.pprof = cmdServer.Flag.Bool("volume.pprof", false, "enable pprof http handlers. precludes --memprofile and --cpuprofile") s3Options.port = cmdServer.Flag.Int("s3.port", 8333, "s3 server http listen port") - s3Options.domainName = cmdServer.Flag.String("s3.domainName", "", "suffix of the host name, {bucket}.{domainName}") + s3Options.domainName = cmdServer.Flag.String("s3.domainName", "", "suffix of the host name in comma separated list, {bucket}.{domainName}") s3Options.tlsPrivateKey = cmdServer.Flag.String("s3.key.file", "", "path to the TLS private key file") s3Options.tlsCertificate = cmdServer.Flag.String("s3.cert.file", "", "path to the TLS certificate file") s3Options.config = cmdServer.Flag.String("s3.config", "", "path to the config file") From e73d6c9526078951bd5f3cc848b1bc61573d948c Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Wed, 21 Oct 2020 23:48:07 -0700 Subject: [PATCH 10/11] able to print partial file id address https://github.com/chrislusf/seaweedfs/issues/1552 --- weed/shell/command_volume_fsck.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/weed/shell/command_volume_fsck.go b/weed/shell/command_volume_fsck.go index 4b3568acb..4031cd237 100644 --- a/weed/shell/command_volume_fsck.go +++ b/weed/shell/command_volume_fsck.go @@ -102,6 +102,12 @@ func (c *commandVolumeFsck) Do(args []string, commandEnv *CommandEnv, writer io. totalOrphanChunkCount += uint64(len(orphanFileIds)) totalOrphanDataSize += orphanDataSize + if *verbose { + for _, fid := range orphanFileIds { + fmt.Fprintf(writer, "%sxxxxxxxx\n", fid) + } + } + if *applyPurging && len(orphanFileIds) > 0 { if vinfo.isEcVolume { fmt.Fprintf(writer, "Skip purging for Erasure Coded volumes.\n") From 5179e559f74cf7aed562f785e40bef46da3191bf Mon Sep 17 00:00:00 2001 From: Chris Lu Date: Thu, 22 Oct 2020 00:35:46 -0700 Subject: [PATCH 11/11] skip empty logs --- weed/filer/filer_notify.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/weed/filer/filer_notify.go b/weed/filer/filer_notify.go index 8719cf5b5..40755e6a7 100644 --- a/weed/filer/filer_notify.go +++ b/weed/filer/filer_notify.go @@ -83,6 +83,10 @@ func (f *Filer) logMetaEvent(ctx context.Context, fullpath string, eventNotifica func (f *Filer) logFlushFunc(startTime, stopTime time.Time, buf []byte) { + if len(buf) == 0 { + return + } + startTime, stopTime = startTime.UTC(), stopTime.UTC() targetFile := fmt.Sprintf("%s/%04d-%02d-%02d/%02d-%02d.segment", SystemLogDir,