From 1a41691b4c17b36b8ad39491ce579547e1ee4c04 Mon Sep 17 00:00:00 2001
From: liubaojiang <1838095916@qq.com>
Date: Fri, 20 May 2022 14:33:47 +0800
Subject: [PATCH 01/38] exclude replication from the concurrentUploadLimitMB
---
weed/server/volume_server.go | 2 ++
weed/server/volume_server_handlers.go | 15 ++++++++-------
2 files changed, 10 insertions(+), 7 deletions(-)
diff --git a/weed/server/volume_server.go b/weed/server/volume_server.go
index 477a3709c..e557cf76b 100644
--- a/weed/server/volume_server.go
+++ b/weed/server/volume_server.go
@@ -24,6 +24,7 @@ type VolumeServer struct {
inFlightDownloadDataSize int64
concurrentUploadLimit int64
concurrentDownloadLimit int64
+ inFlightUploadDataLimitCond *sync.Cond
inFlightDownloadDataLimitCond *sync.Cond
SeedMasterNodes []pb.ServerAddress
@@ -84,6 +85,7 @@ func NewVolumeServer(adminMux, publicMux *http.ServeMux, ip string,
fileSizeLimitBytes: int64(fileSizeLimitMB) * 1024 * 1024,
isHeartbeating: true,
stopChan: make(chan bool),
+ inFlightUploadDataLimitCond: sync.NewCond(new(sync.Mutex)),
inFlightDownloadDataLimitCond: sync.NewCond(new(sync.Mutex)),
concurrentUploadLimit: concurrentUploadLimit,
concurrentDownloadLimit: concurrentDownloadLimit,
diff --git a/weed/server/volume_server_handlers.go b/weed/server/volume_server_handlers.go
index 49bc297fb..afd32fed4 100644
--- a/weed/server/volume_server_handlers.go
+++ b/weed/server/volume_server_handlers.go
@@ -1,7 +1,6 @@
package weed_server
import (
- "fmt"
"net/http"
"strconv"
"strings"
@@ -60,13 +59,15 @@ func (vs *VolumeServer) privateStoreHandler(w http.ResponseWriter, r *http.Reque
contentLength := getContentLength(r)
// exclude the replication from the concurrentUploadLimitMB
- if vs.concurrentUploadLimit != 0 && r.URL.Query().Get("type") != "replicate" &&
- atomic.LoadInt64(&vs.inFlightUploadDataSize) > vs.concurrentUploadLimit {
- err := fmt.Errorf("reject because inflight upload data %d > %d", vs.inFlightUploadDataSize, vs.concurrentUploadLimit)
- glog.V(1).Infof("too many requests: %v", err)
- writeJsonError(w, r, http.StatusTooManyRequests, err)
- return
+ if r.URL.Query().Get("type") != "replicate" { //Non-Replication
+ vs.inFlightUploadDataLimitCond.L.Lock()
+ for vs.concurrentUploadLimit != 0 && atomic.LoadInt64(&vs.inFlightUploadDataSize) > vs.concurrentUploadLimit {
+ glog.V(4).Infof("wait because inflight upload data %d > %d", vs.inFlightUploadDataSize, vs.concurrentUploadLimit)
+ vs.inFlightUploadDataLimitCond.Wait()
+ }
+ vs.inFlightUploadDataLimitCond.L.Unlock()
}
+
atomic.AddInt64(&vs.inFlightUploadDataSize, contentLength)
defer func() {
atomic.AddInt64(&vs.inFlightUploadDataSize, -contentLength)
From 71b2e6223e07eaa5d70efdc8ccbe7f39ce6a0169 Mon Sep 17 00:00:00 2001
From: liubaojiang <1838095916@qq.com>
Date: Fri, 20 May 2022 15:19:35 +0800
Subject: [PATCH 02/38] add inFlightUploadDataLimitCond signal
---
weed/server/volume_server_handlers.go | 1 +
1 file changed, 1 insertion(+)
diff --git a/weed/server/volume_server_handlers.go b/weed/server/volume_server_handlers.go
index afd32fed4..c199fa46a 100644
--- a/weed/server/volume_server_handlers.go
+++ b/weed/server/volume_server_handlers.go
@@ -71,6 +71,7 @@ func (vs *VolumeServer) privateStoreHandler(w http.ResponseWriter, r *http.Reque
atomic.AddInt64(&vs.inFlightUploadDataSize, contentLength)
defer func() {
atomic.AddInt64(&vs.inFlightUploadDataSize, -contentLength)
+ vs.inFlightUploadDataLimitCond.Signal()
}()
// processs uploads
From 076e48a6761396034b6c8132278330593ca698c5 Mon Sep 17 00:00:00 2001
From: liubaojiang <1838095916@qq.com>
Date: Fri, 20 May 2022 18:18:20 +0800
Subject: [PATCH 03/38] add inflight upload data wait timeout
---
weed/command/server.go | 1 +
weed/command/volume.go | 5 ++++-
weed/server/volume_server.go | 4 ++++
weed/server/volume_server_handlers.go | 28 +++++++++++++++++++--------
4 files changed, 29 insertions(+), 9 deletions(-)
diff --git a/weed/command/server.go b/weed/command/server.go
index 4b6b6c642..405d432f0 100644
--- a/weed/command/server.go
+++ b/weed/command/server.go
@@ -131,6 +131,7 @@ func init() {
serverOptions.v.pprof = cmdServer.Flag.Bool("volume.pprof", false, "enable pprof http handlers. precludes --memprofile and --cpuprofile")
serverOptions.v.idxFolder = cmdServer.Flag.String("volume.dir.idx", "", "directory to store .idx files")
serverOptions.v.enableTcp = cmdServer.Flag.Bool("volume.tcp", false, "<exprimental> enable tcp port")
+ serverOptions.v.inflightUploadDataTimeout = cmdServer.Flag.Duration("volume.inflightUploadDataTimeout", 60*time.Second, "inflight upload data wait timeout of volume servers")
s3Options.port = cmdServer.Flag.Int("s3.port", 8333, "s3 server http listen port")
s3Options.portGrpc = cmdServer.Flag.Int("s3.port.grpc", 0, "s3 server grpc listen port")
diff --git a/weed/command/volume.go b/weed/command/volume.go
index b1455352c..158bdf162 100644
--- a/weed/command/volume.go
+++ b/weed/command/volume.go
@@ -65,7 +65,8 @@ type VolumeServerOptions struct {
preStopSeconds *int
metricsHttpPort *int
// pulseSeconds *int
- enableTcp *bool
+ enableTcp *bool
+ inflightUploadDataTimeout *time.Duration
}
func init() {
@@ -96,6 +97,7 @@ func init() {
v.metricsHttpPort = cmdVolume.Flag.Int("metricsPort", 0, "Prometheus metrics listen port")
v.idxFolder = cmdVolume.Flag.String("dir.idx", "", "directory to store .idx files")
v.enableTcp = cmdVolume.Flag.Bool("tcp", false, "<experimental> enable tcp port")
+ v.inflightUploadDataTimeout = cmdVolume.Flag.Duration("inflightUploadDataTimeout", 60*time.Second, "inflight upload data wait timeout of volume servers")
}
var cmdVolume = &Command{
@@ -244,6 +246,7 @@ func (v VolumeServerOptions) startVolumeServer(volumeFolders, maxVolumeCounts, v
*v.fileSizeLimitMB,
int64(*v.concurrentUploadLimitMB)*1024*1024,
int64(*v.concurrentDownloadLimitMB)*1024*1024,
+ *v.inflightUploadDataTimeout,
)
// starting grpc server
grpcS := v.startGrpcService(volumeServer)
diff --git a/weed/server/volume_server.go b/weed/server/volume_server.go
index e557cf76b..abb30229a 100644
--- a/weed/server/volume_server.go
+++ b/weed/server/volume_server.go
@@ -3,6 +3,7 @@ package weed_server
import (
"net/http"
"sync"
+ "time"
"github.com/chrislusf/seaweedfs/weed/pb"
"github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb"
@@ -26,6 +27,7 @@ type VolumeServer struct {
concurrentDownloadLimit int64
inFlightUploadDataLimitCond *sync.Cond
inFlightDownloadDataLimitCond *sync.Cond
+ inflightUploadDataTimeout time.Duration
SeedMasterNodes []pb.ServerAddress
currentMaster pb.ServerAddress
@@ -61,6 +63,7 @@ func NewVolumeServer(adminMux, publicMux *http.ServeMux, ip string,
fileSizeLimitMB int,
concurrentUploadLimit int64,
concurrentDownloadLimit int64,
+ inflightUploadDataTimeout time.Duration,
) *VolumeServer {
v := util.GetViper()
@@ -89,6 +92,7 @@ func NewVolumeServer(adminMux, publicMux *http.ServeMux, ip string,
inFlightDownloadDataLimitCond: sync.NewCond(new(sync.Mutex)),
concurrentUploadLimit: concurrentUploadLimit,
concurrentDownloadLimit: concurrentDownloadLimit,
+ inflightUploadDataTimeout: inflightUploadDataTimeout,
}
vs.SeedMasterNodes = masterNodes
diff --git a/weed/server/volume_server_handlers.go b/weed/server/volume_server_handlers.go
index c199fa46a..aa231e650 100644
--- a/weed/server/volume_server_handlers.go
+++ b/weed/server/volume_server_handlers.go
@@ -1,10 +1,12 @@
package weed_server
import (
+ "fmt"
"net/http"
"strconv"
"strings"
"sync/atomic"
+ "time"
"github.com/chrislusf/seaweedfs/weed/util"
@@ -55,22 +57,32 @@ func (vs *VolumeServer) privateStoreHandler(w http.ResponseWriter, r *http.Reque
vs.guard.WhiteList(vs.DeleteHandler)(w, r)
case "PUT", "POST":
- // wait until in flight data is less than the limit
contentLength := getContentLength(r)
-
+ startTime := time.Now()
+ vs.inFlightUploadDataLimitCond.L.Lock()
// exclude the replication from the concurrentUploadLimitMB
- if r.URL.Query().Get("type") != "replicate" { //Non-Replication
- vs.inFlightUploadDataLimitCond.L.Lock()
- for vs.concurrentUploadLimit != 0 && atomic.LoadInt64(&vs.inFlightUploadDataSize) > vs.concurrentUploadLimit {
+ if r.URL.Query().Get("type") != "replicate" {
+ for vs.concurrentUploadLimit != 0 && vs.inFlightUploadDataSize > vs.concurrentUploadLimit {
+ //wait timeout
+ if startTime.Add(vs.inflightUploadDataTimeout).Before(time.Now()) {
+ err := fmt.Errorf("reject because inflight upload data %d > %d, and wait timeout", vs.inFlightUploadDataSize, vs.concurrentUploadLimit)
+ vs.inFlightUploadDataLimitCond.L.Unlock()
+ glog.V(1).Infof("too many requests: %v", err)
+ writeJsonError(w, r, http.StatusTooManyRequests, err)
+ return
+ }
+
glog.V(4).Infof("wait because inflight upload data %d > %d", vs.inFlightUploadDataSize, vs.concurrentUploadLimit)
vs.inFlightUploadDataLimitCond.Wait()
}
- vs.inFlightUploadDataLimitCond.L.Unlock()
}
+ vs.inFlightUploadDataSize += contentLength
+ vs.inFlightUploadDataLimitCond.L.Unlock()
- atomic.AddInt64(&vs.inFlightUploadDataSize, contentLength)
defer func() {
- atomic.AddInt64(&vs.inFlightUploadDataSize, -contentLength)
+ vs.inFlightUploadDataLimitCond.L.Lock()
+ vs.inFlightUploadDataSize -= contentLength
+ vs.inFlightUploadDataLimitCond.L.Unlock()
vs.inFlightUploadDataLimitCond.Signal()
}()
From f0ee3e6f2129eb2637ea0fdb18540f18147bf474 Mon Sep 17 00:00:00 2001
From: liubaojiang <1838095916@qq.com>
Date: Tue, 31 May 2022 09:40:25 +0800
Subject: [PATCH 04/38] reduce the scope of inFlightUploadDataLimitCond lock
---
weed/server/volume_server_handlers.go | 18 +++++++-----------
1 file changed, 7 insertions(+), 11 deletions(-)
diff --git a/weed/server/volume_server_handlers.go b/weed/server/volume_server_handlers.go
index aa231e650..7e4c11fed 100644
--- a/weed/server/volume_server_handlers.go
+++ b/weed/server/volume_server_handlers.go
@@ -58,31 +58,27 @@ func (vs *VolumeServer) privateStoreHandler(w http.ResponseWriter, r *http.Reque
case "PUT", "POST":
contentLength := getContentLength(r)
- startTime := time.Now()
- vs.inFlightUploadDataLimitCond.L.Lock()
// exclude the replication from the concurrentUploadLimitMB
if r.URL.Query().Get("type") != "replicate" {
+ startTime := time.Now()
+ vs.inFlightUploadDataLimitCond.L.Lock()
for vs.concurrentUploadLimit != 0 && vs.inFlightUploadDataSize > vs.concurrentUploadLimit {
- //wait timeout
+ //wait timeout check
if startTime.Add(vs.inflightUploadDataTimeout).Before(time.Now()) {
- err := fmt.Errorf("reject because inflight upload data %d > %d, and wait timeout", vs.inFlightUploadDataSize, vs.concurrentUploadLimit)
vs.inFlightUploadDataLimitCond.L.Unlock()
+ err := fmt.Errorf("reject because inflight upload data %d > %d, and wait timeout", vs.inFlightUploadDataSize, vs.concurrentUploadLimit)
glog.V(1).Infof("too many requests: %v", err)
writeJsonError(w, r, http.StatusTooManyRequests, err)
return
}
-
glog.V(4).Infof("wait because inflight upload data %d > %d", vs.inFlightUploadDataSize, vs.concurrentUploadLimit)
vs.inFlightUploadDataLimitCond.Wait()
}
+ vs.inFlightUploadDataLimitCond.L.Unlock()
}
- vs.inFlightUploadDataSize += contentLength
- vs.inFlightUploadDataLimitCond.L.Unlock()
-
+ atomic.AddInt64(&vs.inFlightUploadDataSize, contentLength)
defer func() {
- vs.inFlightUploadDataLimitCond.L.Lock()
- vs.inFlightUploadDataSize -= contentLength
- vs.inFlightUploadDataLimitCond.L.Unlock()
+ atomic.AddInt64(&vs.inFlightUploadDataSize, -contentLength)
vs.inFlightUploadDataLimitCond.Signal()
}()
From 2da2ff00a3f346335c1d61324697e72c03e4bf3d Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 13 Jun 2022 15:19:43 +0000
Subject: [PATCH 05/38] Bump google.golang.org/api from 0.82.0 to 0.83.0
Bumps [google.golang.org/api](https://github.com/googleapis/google-api-go-client) from 0.82.0 to 0.83.0.
- [Release notes](https://github.com/googleapis/google-api-go-client/releases)
- [Changelog](https://github.com/googleapis/google-api-go-client/blob/main/CHANGES.md)
- [Commits](https://github.com/googleapis/google-api-go-client/compare/v0.82.0...v0.83.0)
---
updated-dependencies:
- dependency-name: google.golang.org/api
dependency-type: direct:production
update-type: version-update:semver-minor
...
Signed-off-by: dependabot[bot] <support@github.com>
---
go.mod | 8 ++++----
go.sum | 18 ++++++++++--------
2 files changed, 14 insertions(+), 12 deletions(-)
diff --git a/go.mod b/go.mod
index 2fbff4d37..0e3a4cfce 100644
--- a/go.mod
+++ b/go.mod
@@ -123,15 +123,15 @@ require (
golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4 // indirect
golang.org/x/exp v0.0.0-20220414153411-bcd21879b8fd
golang.org/x/image v0.0.0-20200119044424-58c23975cae1
- golang.org/x/net v0.0.0-20220526153639-5463443f8c37
+ golang.org/x/net v0.0.0-20220607020251-c690dde0001d
golang.org/x/oauth2 v0.0.0-20220524215830-622c5d57e401 // indirect
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a
golang.org/x/text v0.3.7 // indirect
golang.org/x/tools v0.1.8-0.20211029000441-d6a9af8af023
golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df // indirect
- google.golang.org/api v0.82.0
+ google.golang.org/api v0.83.0
google.golang.org/appengine v1.6.7 // indirect
- google.golang.org/genproto v0.0.0-20220527130721-00d5c0f3be58 // indirect
+ google.golang.org/genproto v0.0.0-20220602131408-e326c6e8e9c8 // indirect
google.golang.org/grpc v1.47.0
google.golang.org/protobuf v1.28.0
gopkg.in/inf.v0 v0.9.1 // indirect
@@ -215,7 +215,7 @@ require (
go.uber.org/multierr v1.8.0 // indirect
go.uber.org/zap v1.21.0 // indirect
golang.org/x/mod v0.6.0-dev.0.20211013180041-c96bc1413d57 // indirect
- golang.org/x/sync v0.0.0-20220513210516-0976fa681c29 // indirect
+ golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f // indirect
gopkg.in/ini.v1 v1.66.4 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
diff --git a/go.sum b/go.sum
index 6fc6a2787..e5101b0d5 100644
--- a/go.sum
+++ b/go.sum
@@ -1059,8 +1059,8 @@ golang.org/x/net v0.0.0-20220412020605-290c469a71a5/go.mod h1:CfG3xpIq0wQ8r1q4Su
golang.org/x/net v0.0.0-20220420153159-1850ba15e1be/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
golang.org/x/net v0.0.0-20220520000938-2e3eb7b945c2/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
-golang.org/x/net v0.0.0-20220526153639-5463443f8c37 h1:lUkvobShwKsOesNfWWlCS5q7fnbG1MEliIzwu886fn8=
-golang.org/x/net v0.0.0-20220526153639-5463443f8c37/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
+golang.org/x/net v0.0.0-20220607020251-c690dde0001d h1:4SFsTMi4UahlKoloni7L4eYzhFRifURQLw+yv0QDCx8=
+golang.org/x/net v0.0.0-20220607020251-c690dde0001d/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -1095,8 +1095,8 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20220513210516-0976fa681c29 h1:w8s32wxx3sY+OjLlv9qltkLU5yvJzxjjgiHWLjdIcw4=
-golang.org/x/sync v0.0.0-20220513210516-0976fa681c29/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f h1:Ax0t5p6N38Ga0dThY21weqDEyz2oklo4IvDkpigvkD8=
+golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -1334,8 +1334,8 @@ google.golang.org/api v0.74.0/go.mod h1:ZpfMZOVRMywNyvJFeqL9HRWBgAuRfSjJFpe9QtRR
google.golang.org/api v0.75.0/go.mod h1:pU9QmyHLnzlpar1Mjt4IbapUCy8J+6HD6GeELN69ljA=
google.golang.org/api v0.76.0/go.mod h1:pU9QmyHLnzlpar1Mjt4IbapUCy8J+6HD6GeELN69ljA=
google.golang.org/api v0.78.0/go.mod h1:1Sg78yoMLOhlQTeF+ARBoytAcH1NNyyl390YMy6rKmw=
-google.golang.org/api v0.82.0 h1:h6EGeZuzhoKSS7BUznzkW+2wHZ+4Ubd6rsVvvh3dRkw=
-google.golang.org/api v0.82.0/go.mod h1:Ld58BeTlL9DIYr2M2ajvoSqmGLei0BMn+kVBmkam1os=
+google.golang.org/api v0.83.0 h1:pMvST+6v+46Gabac4zlJlalxZjCeRcepwg2EdBU+nCc=
+google.golang.org/api v0.83.0/go.mod h1:CNywQoj/AfhTw26ZWAa6LwOv+6WFxHmeLPZq2uncLZk=
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
@@ -1438,8 +1438,8 @@ google.golang.org/genproto v0.0.0-20220426171045-31bebdecfb46/go.mod h1:8w6bsBMX
google.golang.org/genproto v0.0.0-20220429170224-98d788798c3e/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo=
google.golang.org/genproto v0.0.0-20220505152158-f39f71e6c8f3/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4=
google.golang.org/genproto v0.0.0-20220518221133-4f43b3371335/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4=
-google.golang.org/genproto v0.0.0-20220527130721-00d5c0f3be58 h1:a221mAAEAzq4Lz6ZWRkcS8ptb2mxoxYSt4N68aRyQHM=
-google.golang.org/genproto v0.0.0-20220527130721-00d5c0f3be58/go.mod h1:yKyY4AMRwFiC8yMMNaMi+RkCnjZJt9LoWuvhXjMs+To=
+google.golang.org/genproto v0.0.0-20220602131408-e326c6e8e9c8 h1:qRu95HZ148xXw+XeZ3dvqe85PxH4X8+jIo0iRPKcEnM=
+google.golang.org/genproto v0.0.0-20220602131408-e326c6e8e9c8/go.mod h1:yKyY4AMRwFiC8yMMNaMi+RkCnjZJt9LoWuvhXjMs+To=
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
@@ -1513,6 +1513,8 @@ gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.0-20220512140231-539c8e751b99/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
From d4156e4f26255cd872e5d054f9ebf48b5652f4a0 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 13 Jun 2022 15:20:00 +0000
Subject: [PATCH 06/38] Bump github.com/ydb-platform/ydb-go-sdk/v3 from 3.25.3
to 3.26.10
Bumps [github.com/ydb-platform/ydb-go-sdk/v3](https://github.com/ydb-platform/ydb-go-sdk) from 3.25.3 to 3.26.10.
- [Release notes](https://github.com/ydb-platform/ydb-go-sdk/releases)
- [Changelog](https://github.com/ydb-platform/ydb-go-sdk/blob/master/CHANGELOG.md)
- [Commits](https://github.com/ydb-platform/ydb-go-sdk/compare/v3.25.3...v3.26.10)
---
updated-dependencies:
- dependency-name: github.com/ydb-platform/ydb-go-sdk/v3
dependency-type: direct:production
update-type: version-update:semver-minor
...
Signed-off-by: dependabot[bot] <support@github.com>
---
go.mod | 5 +++--
go.sum | 11 +++++++++--
2 files changed, 12 insertions(+), 4 deletions(-)
diff --git a/go.mod b/go.mod
index 2fbff4d37..f1463cf30 100644
--- a/go.mod
+++ b/go.mod
@@ -155,7 +155,7 @@ require (
github.com/hashicorp/raft v1.3.9
github.com/hashicorp/raft-boltdb v0.0.0-20220329195025-15018e9b97e0
github.com/ydb-platform/ydb-go-sdk-auth-environ v0.1.2
- github.com/ydb-platform/ydb-go-sdk/v3 v3.25.3
+ github.com/ydb-platform/ydb-go-sdk/v3 v3.26.10
)
require (
@@ -190,6 +190,7 @@ require (
github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect
github.com/jcmturner/goidentity/v6 v6.0.1 // indirect
github.com/jcmturner/rpc/v2 v2.0.3 // indirect
+ github.com/jonboulle/clockwork v0.2.2 // indirect
github.com/josharian/intern v1.0.0 // indirect
github.com/klauspost/cpuid/v2 v2.0.6 // indirect
github.com/mattn/go-colorable v0.1.12 // indirect
@@ -206,7 +207,7 @@ require (
github.com/subosito/gotenv v1.3.0 // indirect
github.com/tinylib/msgp v1.1.6 // indirect
github.com/yandex-cloud/go-genproto v0.0.0-20211115083454-9ca41db5ed9e // indirect
- github.com/ydb-platform/ydb-go-genproto v0.0.0-20220203104745-929cf9c248bc // indirect
+ github.com/ydb-platform/ydb-go-genproto v0.0.0-20220531094121-36ca6bddb9f7 // indirect
github.com/ydb-platform/ydb-go-yc v0.8.3 // indirect
github.com/ydb-platform/ydb-go-yc-metadata v0.5.2 // indirect
go.etcd.io/etcd/api/v3 v3.5.4 // indirect
diff --git a/go.sum b/go.sum
index 6fc6a2787..a523b1204 100644
--- a/go.sum
+++ b/go.sum
@@ -579,6 +579,8 @@ github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHW
github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg=
+github.com/jonboulle/clockwork v0.2.2 h1:UOGuzwb1PwsrDAObMuhUnj0p5ULPj8V/xJ7Kx9qUBdQ=
+github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8=
github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4=
@@ -868,12 +870,14 @@ github.com/xdg-go/stringprep v1.0.3 h1:kdwGpVNwPFtjs98xCGkHjQtGKh86rDcRZN17QEMCO
github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8=
github.com/yandex-cloud/go-genproto v0.0.0-20211115083454-9ca41db5ed9e h1:9LPdmD1vqadsDQUva6t2O9MbnyvoOgo8nFNPaOIH5U8=
github.com/yandex-cloud/go-genproto v0.0.0-20211115083454-9ca41db5ed9e/go.mod h1:HEUYX/p8966tMUHHT+TsS0hF/Ca/NYwqprC5WXSDMfE=
-github.com/ydb-platform/ydb-go-genproto v0.0.0-20220203104745-929cf9c248bc h1:xvTP0fhYNm+Ws+xC34jzF9EdorPUKkucJr0TyybqVSk=
github.com/ydb-platform/ydb-go-genproto v0.0.0-20220203104745-929cf9c248bc/go.mod h1:cc138nptTn9eKptCQl/grxP6pBKpo/bnXDiOxuVZtps=
+github.com/ydb-platform/ydb-go-genproto v0.0.0-20220531094121-36ca6bddb9f7 h1:S3bwscnat3pa188pvEuXDUstxPjCnGpn5bZJuRMkL+g=
+github.com/ydb-platform/ydb-go-genproto v0.0.0-20220531094121-36ca6bddb9f7/go.mod h1:cc138nptTn9eKptCQl/grxP6pBKpo/bnXDiOxuVZtps=
github.com/ydb-platform/ydb-go-sdk-auth-environ v0.1.2 h1:EYSI1kulnHb0H0zt3yOw4cRj4ABMSMGwNe43D+fX7e4=
github.com/ydb-platform/ydb-go-sdk-auth-environ v0.1.2/go.mod h1:Xfjce+VMU9yJVr1lj60yK2fFPWjB4jr/4cp3K7cjzi4=
-github.com/ydb-platform/ydb-go-sdk/v3 v3.25.3 h1:yyMw+sTDSqGVDG8ivAw8V/kXfaulXsgftSG+4nCmseA=
github.com/ydb-platform/ydb-go-sdk/v3 v3.25.3/go.mod h1:PFizF/vJsdAgEwjK3DVSBD52kdmRkWfSIS2q2pA+e88=
+github.com/ydb-platform/ydb-go-sdk/v3 v3.26.10 h1:2M6L2NX8l103qNDM1i97gh81Y2EJEAEyON3/N5zQV14=
+github.com/ydb-platform/ydb-go-sdk/v3 v3.26.10/go.mod h1:1U2nGytADgY4/U7OykO3LfzeS3/zz4zqg7wlUA7XHfc=
github.com/ydb-platform/ydb-go-yc v0.8.3 h1:92UUUMsfvtMl6mho8eQ9lbkiPrF3a9CT+RrVRAKNRwo=
github.com/ydb-platform/ydb-go-yc v0.8.3/go.mod h1:zUolAFGzJ5XG8uwiseTLr9Lapm7L7hdVdZgLSuv9FXE=
github.com/ydb-platform/ydb-go-yc-metadata v0.5.2 h1:nMtixUijP0Z7iHJNT9fOL+dbmEzZxqU6Xk87ll7hqXg=
@@ -1513,7 +1517,10 @@ gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.0-20220512140231-539c8e751b99/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
From a6470579690a37318e3d49846c6c71dae31d8dfe Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 13 Jun 2022 15:20:05 +0000
Subject: [PATCH 07/38] Bump github.com/linxGnu/grocksdb from 1.7.2 to 1.7.3
Bumps [github.com/linxGnu/grocksdb](https://github.com/linxGnu/grocksdb) from 1.7.2 to 1.7.3.
- [Release notes](https://github.com/linxGnu/grocksdb/releases)
- [Commits](https://github.com/linxGnu/grocksdb/compare/v1.7.2...v1.7.3)
---
updated-dependencies:
- dependency-name: github.com/linxGnu/grocksdb
dependency-type: direct:production
update-type: version-update:semver-patch
...
Signed-off-by: dependabot[bot] <support@github.com>
---
go.mod | 2 +-
go.sum | 7 ++++---
2 files changed, 5 insertions(+), 4 deletions(-)
diff --git a/go.mod b/go.mod
index 2fbff4d37..4afb13b29 100644
--- a/go.mod
+++ b/go.mod
@@ -68,7 +68,7 @@ require (
github.com/klauspost/reedsolomon v1.9.16
github.com/kurin/blazer v0.5.3
github.com/lib/pq v1.10.6
- github.com/linxGnu/grocksdb v1.7.2
+ github.com/linxGnu/grocksdb v1.7.3
github.com/magiconair/properties v1.8.6 // indirect
github.com/mailru/easyjson v0.7.7 // indirect
github.com/mattn/go-ieproxy v0.0.3 // indirect
diff --git a/go.sum b/go.sum
index 6fc6a2787..1eef8a2a7 100644
--- a/go.sum
+++ b/go.sum
@@ -636,8 +636,8 @@ github.com/lib/pq v1.10.2/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
github.com/lib/pq v1.10.4/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
github.com/lib/pq v1.10.6 h1:jbk+ZieJ0D7EVGJYpL9QTz7/YW6UHbmdnZWYyK5cdBs=
github.com/lib/pq v1.10.6/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
-github.com/linxGnu/grocksdb v1.7.2 h1:5SVU++WAyOzk4jG0pXrI9plvZEPRCRbh4UrDhAxrktM=
-github.com/linxGnu/grocksdb v1.7.2/go.mod h1:3tKVazPI6orwnNtFAOSRZwhi6bMOkc3+kREb8qpKaSE=
+github.com/linxGnu/grocksdb v1.7.3 h1:S9XiU4FviunvjNdNG+kWe2BoOy/2EKZSdDyeGmL0vDs=
+github.com/linxGnu/grocksdb v1.7.3/go.mod h1:G4zrMNj2CP2aCXF61jbmZH81tu+kU3qU4rYpOU8WOL8=
github.com/magiconair/properties v1.8.6 h1:5ibWZ6iY0NctNGWo87LalDlEZ6R41TqbbDamhfG/Qzo=
github.com/magiconair/properties v1.8.6/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60=
github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
@@ -1513,7 +1513,8 @@ gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
-gopkg.in/yaml.v3 v3.0.0-20220512140231-539c8e751b99/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
From ba16477cef4fecadb9f26ef9c366a9e2fc8168bd Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 13 Jun 2022 15:20:11 +0000
Subject: [PATCH 08/38] Bump github.com/Shopify/sarama from 1.34.0 to 1.34.1
Bumps [github.com/Shopify/sarama](https://github.com/Shopify/sarama) from 1.34.0 to 1.34.1.
- [Release notes](https://github.com/Shopify/sarama/releases)
- [Changelog](https://github.com/Shopify/sarama/blob/main/CHANGELOG.md)
- [Commits](https://github.com/Shopify/sarama/compare/v1.34.0...v1.34.1)
---
updated-dependencies:
- dependency-name: github.com/Shopify/sarama
dependency-type: direct:production
update-type: version-update:semver-patch
...
Signed-off-by: dependabot[bot] <support@github.com>
---
go.mod | 4 ++--
go.sum | 15 +++++++++------
2 files changed, 11 insertions(+), 8 deletions(-)
diff --git a/go.mod b/go.mod
index 2fbff4d37..a86779642 100644
--- a/go.mod
+++ b/go.mod
@@ -9,7 +9,7 @@ require (
github.com/Azure/azure-pipeline-go v0.2.3
github.com/Azure/azure-storage-blob-go v0.15.0
github.com/OneOfOne/xxhash v1.2.8
- github.com/Shopify/sarama v1.34.0
+ github.com/Shopify/sarama v1.34.1
github.com/aws/aws-sdk-go v1.44.14
github.com/beorn7/perks v1.0.1 // indirect
github.com/buraksezer/consistent v0.0.0-20191006190839-693edf70fd72
@@ -64,7 +64,7 @@ require (
github.com/json-iterator/go v1.1.12
github.com/karlseguin/ccache/v2 v2.0.8
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
- github.com/klauspost/compress v1.15.1 // indirect
+ github.com/klauspost/compress v1.15.6 // indirect
github.com/klauspost/reedsolomon v1.9.16
github.com/kurin/blazer v0.5.3
github.com/lib/pq v1.10.6
diff --git a/go.sum b/go.sum
index 6fc6a2787..496670f16 100644
--- a/go.sum
+++ b/go.sum
@@ -128,10 +128,10 @@ github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0
github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
github.com/OneOfOne/xxhash v1.2.8 h1:31czK/TI9sNkxIKfaUfGlU47BAxQ0ztGgd9vPyqimf8=
github.com/OneOfOne/xxhash v1.2.8/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q=
-github.com/Shopify/sarama v1.34.0 h1:j4zTaFHFnfvuV2fdLZyXqIg0Tu4Mzl9f064Z5/H+o4o=
-github.com/Shopify/sarama v1.34.0/go.mod h1:V2ceE9UupUf4/oP1Z38SI49fAnD0/MtkqDDHvolIeeQ=
-github.com/Shopify/toxiproxy/v2 v2.3.0 h1:62YkpiP4bzdhKMH+6uC5E95y608k3zDwdzuBMsnn3uQ=
-github.com/Shopify/toxiproxy/v2 v2.3.0/go.mod h1:KvQTtB6RjCJY4zqNJn7C7JDFgsG5uoHYDirfUfpIm0c=
+github.com/Shopify/sarama v1.34.1 h1:pVCQO7BMAK3s1jWhgi5v1W6lwZ6Veiekfc2vsgRS06Y=
+github.com/Shopify/sarama v1.34.1/go.mod h1:NZSNswsnStpq8TUdFaqnpXm2Do6KRzTIjdBdVlL1YRM=
+github.com/Shopify/toxiproxy/v2 v2.4.0 h1:O1e4Jfvr/hefNTNu+8VtdEG5lSeamJRo4aKhMOKNM64=
+github.com/Shopify/toxiproxy/v2 v2.4.0/go.mod h1:3ilnjng821bkozDRxNoo64oI/DKqM+rOyJzb564+bvg=
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
@@ -603,9 +603,9 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o
github.com/klauspost/compress v1.10.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
github.com/klauspost/compress v1.13.4/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg=
github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
-github.com/klauspost/compress v1.15.0/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
-github.com/klauspost/compress v1.15.1 h1:y9FcTHGyrebwfP0ZZqFiaxTaiDnUrGkJkI+f583BL1A=
github.com/klauspost/compress v1.15.1/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
+github.com/klauspost/compress v1.15.6 h1:6D9PcO8QWu0JyaQ2zUMmu16T1T+zjjEpP91guRsvDfY=
+github.com/klauspost/compress v1.15.6/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
github.com/klauspost/cpuid/v2 v2.0.6 h1:dQ5ueTiftKxp0gyjKSx5+8BtPWkyQbd95m8Gys/RarI=
github.com/klauspost/cpuid/v2 v2.0.6/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
github.com/klauspost/reedsolomon v1.9.16 h1:mR0AwphBwqFv/I3B9AHtNKvzuowI1vrj8/3UX4XRmHA=
@@ -745,6 +745,7 @@ github.com/prometheus/client_golang v1.4.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3O
github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M=
github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0=
github.com/prometheus/client_golang v1.11.1/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0=
+github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY=
github.com/prometheus/client_golang v1.12.2 h1:51L9cDoUHVrXx4zWYlcLQIZ+d+VXHgqnYKkIuq4g/34=
github.com/prometheus/client_golang v1.12.2/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY=
github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
@@ -1513,6 +1514,8 @@ gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.0-20220512140231-539c8e751b99/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
From 1d7c0e65a0a2097147c457a251a6556faf42b304 Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Mon, 13 Jun 2022 15:20:23 +0000
Subject: [PATCH 09/38] Bump github.com/aws/aws-sdk-go from 1.44.14 to 1.44.32
Bumps [github.com/aws/aws-sdk-go](https://github.com/aws/aws-sdk-go) from 1.44.14 to 1.44.32.
- [Release notes](https://github.com/aws/aws-sdk-go/releases)
- [Changelog](https://github.com/aws/aws-sdk-go/blob/main/CHANGELOG.md)
- [Commits](https://github.com/aws/aws-sdk-go/compare/v1.44.14...v1.44.32)
---
updated-dependencies:
- dependency-name: github.com/aws/aws-sdk-go
dependency-type: direct:production
update-type: version-update:semver-patch
...
Signed-off-by: dependabot[bot] <support@github.com>
---
go.mod | 2 +-
go.sum | 6 ++++--
2 files changed, 5 insertions(+), 3 deletions(-)
diff --git a/go.mod b/go.mod
index 2fbff4d37..9485dc401 100644
--- a/go.mod
+++ b/go.mod
@@ -10,7 +10,7 @@ require (
github.com/Azure/azure-storage-blob-go v0.15.0
github.com/OneOfOne/xxhash v1.2.8
github.com/Shopify/sarama v1.34.0
- github.com/aws/aws-sdk-go v1.44.14
+ github.com/aws/aws-sdk-go v1.44.32
github.com/beorn7/perks v1.0.1 // indirect
github.com/buraksezer/consistent v0.0.0-20191006190839-693edf70fd72
github.com/bwmarrin/snowflake v0.3.0
diff --git a/go.sum b/go.sum
index 6fc6a2787..71b3d3f2b 100644
--- a/go.sum
+++ b/go.sum
@@ -150,8 +150,8 @@ github.com/armon/go-metrics v0.3.10/go.mod h1:4O98XIr/9W0sxpJ8UaYkvjk10Iff7SnFrb
github.com/aws/aws-sdk-go v1.15.27/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZoCYDt7FT0=
github.com/aws/aws-sdk-go v1.37.0/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro=
github.com/aws/aws-sdk-go v1.43.31/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo=
-github.com/aws/aws-sdk-go v1.44.14 h1:qd7/muV1rElsbvkK9D1nHUzBoDlEw2etfeo4IE82eSQ=
-github.com/aws/aws-sdk-go v1.44.14/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo=
+github.com/aws/aws-sdk-go v1.44.32 h1:x5hBtpY/02sgRL158zzTclcCLwh3dx3YlSl1rAH4Op0=
+github.com/aws/aws-sdk-go v1.44.32/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo=
github.com/aws/aws-sdk-go-v2 v1.16.2 h1:fqlCk6Iy3bnCumtrLz9r3mJ/2gUT0pJ0wLFVIdWh+JA=
github.com/aws/aws-sdk-go-v2 v1.16.2/go.mod h1:ytwTPBG6fXTZLxxeeCCWj2/EMYp/xDUgX+OET6TLNNU=
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.1/go.mod h1:n8Bs1ElDD2wJ9kCRTczA83gYbBmjSwZp3umc6zF4EeM=
@@ -1513,6 +1513,8 @@ gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.0-20220512140231-539c8e751b99/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
From 2b159c331d5ec9a09aae831725fbb02238abe9dd Mon Sep 17 00:00:00 2001
From: Konstantin Lebedev <9497591+kmlebedev@users.noreply.github.com>
Date: Tue, 14 Jun 2022 13:16:09 +0500
Subject: [PATCH 10/38] fix build container
---
.github/workflows/container_latest.yml | 2 +-
.github/workflows/container_release1.yml | 2 +-
.github/workflows/container_release2.yml | 2 +-
.github/workflows/container_release3.yml | 2 +-
.github/workflows/container_release4.yml | 2 +-
.github/workflows/container_release5.yml | 2 +-
6 files changed, 6 insertions(+), 6 deletions(-)
diff --git a/.github/workflows/container_latest.yml b/.github/workflows/container_latest.yml
index 04939b8d6..35dcea714 100644
--- a/.github/workflows/container_latest.yml
+++ b/.github/workflows/container_latest.yml
@@ -4,7 +4,7 @@ on:
push:
tags:
- '*'
- workflow_dispatch: []
+ workflow_dispatch: {}
permissions:
contents: read
diff --git a/.github/workflows/container_release1.yml b/.github/workflows/container_release1.yml
index b8ff81a1b..1bcf768cd 100644
--- a/.github/workflows/container_release1.yml
+++ b/.github/workflows/container_release1.yml
@@ -4,7 +4,7 @@ on:
push:
tags:
- '*'
- workflow_dispatch: []
+ workflow_dispatch: {}
permissions:
contents: read
diff --git a/.github/workflows/container_release2.yml b/.github/workflows/container_release2.yml
index a02ab4f87..c58bb2b40 100644
--- a/.github/workflows/container_release2.yml
+++ b/.github/workflows/container_release2.yml
@@ -4,7 +4,7 @@ on:
push:
tags:
- '*'
- workflow_dispatch: []
+ workflow_dispatch: {}
permissions:
contents: read
diff --git a/.github/workflows/container_release3.yml b/.github/workflows/container_release3.yml
index 104a26ef6..5ff6cd497 100644
--- a/.github/workflows/container_release3.yml
+++ b/.github/workflows/container_release3.yml
@@ -4,7 +4,7 @@ on:
push:
tags:
- '*'
- workflow_dispatch: []
+ workflow_dispatch: {}
permissions:
contents: read
diff --git a/.github/workflows/container_release4.yml b/.github/workflows/container_release4.yml
index 92d776f79..0788abeca 100644
--- a/.github/workflows/container_release4.yml
+++ b/.github/workflows/container_release4.yml
@@ -4,7 +4,7 @@ on:
push:
tags:
- '*'
- workflow_dispatch: []
+ workflow_dispatch: {}
permissions:
contents: read
diff --git a/.github/workflows/container_release5.yml b/.github/workflows/container_release5.yml
index 820527f5b..e536a20b7 100644
--- a/.github/workflows/container_release5.yml
+++ b/.github/workflows/container_release5.yml
@@ -4,7 +4,7 @@ on:
push:
tags:
- '*'
- workflow_dispatch: []
+ workflow_dispatch: {}
permissions:
contents: read
From cc5d7710880491b7c72ca0dbd60008a9c3a84f08 Mon Sep 17 00:00:00 2001
From: Chris Lu <chrislusf@users.noreply.github.com>
Date: Tue, 14 Jun 2022 01:37:28 -0700
Subject: [PATCH 11/38] Update container_dev.yml
---
.github/workflows/container_dev.yml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.github/workflows/container_dev.yml b/.github/workflows/container_dev.yml
index ac93e4537..d8a2312ea 100644
--- a/.github/workflows/container_dev.yml
+++ b/.github/workflows/container_dev.yml
@@ -3,7 +3,7 @@ name: "docker: build dev containers"
on:
push:
branches: [ master ]
- workflow_dispatch: []
+ workflow_dispatch: {}
permissions:
contents: read
From 322dbcf372d75f144b01a17b2a19195fe43b038a Mon Sep 17 00:00:00 2001
From: guol-fnst <goul-fnst@fujitsu.com>
Date: Tue, 14 Jun 2022 19:01:57 +0800
Subject: [PATCH 12/38] add update feature
---
weed/command/command.go | 4 +-
weed/command/update.go | 386 ++++++++++++++++++++++++++++++++++++++++
2 files changed, 389 insertions(+), 1 deletion(-)
create mode 100644 weed/command/update.go
diff --git a/weed/command/command.go b/weed/command/command.go
index dbc18a053..b16de3eba 100644
--- a/weed/command/command.go
+++ b/weed/command/command.go
@@ -2,9 +2,10 @@ package command
import (
"fmt"
- flag "github.com/chrislusf/seaweedfs/weed/util/fla9"
"os"
"strings"
+
+ flag "github.com/chrislusf/seaweedfs/weed/util/fla9"
)
var Commands = []*Command{
@@ -37,6 +38,7 @@ var Commands = []*Command{
cmdServer,
cmdShell,
cmdUpload,
+ cmdUpdate,
cmdVersion,
cmdVolume,
cmdWebDav,
diff --git a/weed/command/update.go b/weed/command/update.go
new file mode 100644
index 000000000..bda904a85
--- /dev/null
+++ b/weed/command/update.go
@@ -0,0 +1,386 @@
+package command
+
+import (
+ "archive/tar"
+ "archive/zip"
+ "bytes"
+ "compress/gzip"
+ "context"
+ "crypto/md5"
+ "encoding/hex"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "io"
+ "io/ioutil"
+ "net/http"
+ "os"
+ "path/filepath"
+ "runtime"
+ "strings"
+ "time"
+
+ "github.com/chrislusf/seaweedfs/weed/glog"
+ "github.com/chrislusf/seaweedfs/weed/util"
+ "golang.org/x/net/context/ctxhttp"
+)
+
+// Release collects data about a single release on GitHub.
+type Release struct {
+ Name string `json:"name"`
+ TagName string `json:"tag_name"`
+ Draft bool `json:"draft"`
+ PreRelease bool `json:"prerelease"`
+ PublishedAt time.Time `json:"published_at"`
+ Assets []Asset `json:"assets"`
+
+ Version string `json:"-"` // set manually in the code
+}
+
+// Asset is a file uploaded and attached to a release.
+type Asset struct {
+ ID int `json:"id"`
+ Name string `json:"name"`
+ URL string `json:"url"`
+}
+
+var (
+ updateOpt UpdateOptions
+)
+
+type UpdateOptions struct {
+ Output *string
+}
+
+func init() {
+ updateOpt.Output = cmdUpdate.Flag.String("output", "weed", "Save the latest weed as `filename`")
+ cmdUpdate.Run = runUpdate
+}
+
+var cmdUpdate = &Command{
+ UsageLine: "update [-output=weed]",
+ Short: "get latest stable version from https://github.com/chrislusf/seaweedfs",
+ Long: `get latest stable version from https://github.com/chrislusf/seaweedfs`,
+}
+
+func runUpdate(cmd *Command, args []string) bool {
+ weedPath := *updateOpt.Output
+ if weedPath == "" {
+ file, err := os.Executable()
+ if err != nil {
+ glog.Fatalf("unable to find executable:%s", err)
+ return false
+ }
+
+ *updateOpt.Output = file
+ }
+
+ fi, err := os.Lstat(*updateOpt.Output)
+ if err != nil {
+ dirname := filepath.Dir(*updateOpt.Output)
+ di, err := os.Lstat(dirname)
+ if err != nil {
+ glog.Fatalf("unable to find directory:%s", dirname)
+ return false
+ }
+ if !di.Mode().IsDir() {
+ glog.Fatalf("output parent path %v is not a directory, use --output to specify a different file path", dirname)
+ return false
+ }
+ } else {
+ if !fi.Mode().IsRegular() {
+ glog.Fatalf("output path %v is not a normal file, use --output to specify a different file path", updateOpt.Output)
+ return false
+ }
+ }
+
+ glog.V(0).Infof("writing weed to %v\n", *updateOpt.Output)
+
+ v, err := downloadLatestStableRelease(context.Background(), *updateOpt.Output)
+ if err != nil {
+ glog.Fatalf("unable to update weed: %v", err)
+ return false
+ }
+
+ glog.V(0).Infof("successfully updated weed to version %v\n", v)
+
+ return true
+}
+
+func downloadLatestStableRelease(ctx context.Context, target string) (version string, err error) {
+ currentVersion := util.VERSION_NUMBER
+ largeDiskSuffix := ""
+ if util.VolumeSizeLimitGB == 8000 {
+ largeDiskSuffix = "_large_disk"
+ }
+
+ rel, err := GitHubLatestRelease(ctx, "chrislusf", "seaweedfs")
+ if err != nil {
+ return "", err
+ }
+
+ if rel.Version == currentVersion {
+ glog.V(0).Infof("weed is up to date\n")
+ return currentVersion, nil
+ }
+
+ glog.V(0).Infof("latest version is %v\n", rel.Version)
+
+ ext := "tar.gz"
+ if runtime.GOOS == "windows" {
+ ext = "zip"
+ }
+
+ suffix := fmt.Sprintf("%s_%s%s.%s", runtime.GOOS, runtime.GOARCH, largeDiskSuffix, ext)
+ md5Filename := fmt.Sprintf("%s.md5", suffix)
+ _, md5Val, err := getGithubDataFile(ctx, rel.Assets, md5Filename)
+ if err != nil {
+ return "", err
+ }
+
+ downloadFilename, buf, err := getGithubDataFile(ctx, rel.Assets, suffix)
+ if err != nil {
+ return "", err
+ }
+
+ md5Ctx := md5.New()
+ md5Ctx.Write(buf)
+ binaryMd5 := md5Ctx.Sum(nil)
+ if hex.EncodeToString(binaryMd5) != string(md5Val[0:32]) {
+ glog.Errorf("md5:'%s' '%s'", hex.EncodeToString(binaryMd5), string(md5Val[0:32]))
+ err = errors.New("binary md5sum doesn't match")
+ return "", err
+ }
+
+ err = extractToFile(buf, downloadFilename, target)
+ if err != nil {
+ return "", err
+ }
+
+ return rel.Version, nil
+}
+
+func (r Release) String() string {
+ return fmt.Sprintf("%v %v, %d assets",
+ r.TagName,
+ r.PublishedAt.Local().Format("2006-01-02 15:04:05"),
+ len(r.Assets))
+}
+
+const githubAPITimeout = 30 * time.Second
+
+// githubError is returned by the GitHub API, e.g. for rate-limiting.
+type githubError struct {
+ Message string
+}
+
+// GitHubLatestRelease uses the GitHub API to get information about the latest
+// release of a repository.
+func GitHubLatestRelease(ctx context.Context, owner, repo string) (Release, error) {
+ ctx, cancel := context.WithTimeout(ctx, githubAPITimeout)
+ defer cancel()
+
+ url := fmt.Sprintf("https://api.github.com/repos/%s/%s/releases/latest", owner, repo)
+ req, err := http.NewRequest(http.MethodGet, url, nil)
+ if err != nil {
+ return Release{}, err
+ }
+
+ // pin API version 3
+ req.Header.Set("Accept", "application/vnd.github.v3+json")
+
+ res, err := ctxhttp.Do(ctx, http.DefaultClient, req)
+ if err != nil {
+ return Release{}, err
+ }
+
+ if res.StatusCode != http.StatusOK {
+ content := res.Header.Get("Content-Type")
+ if strings.Contains(content, "application/json") {
+ // try to decode error message
+ var msg githubError
+ jerr := json.NewDecoder(res.Body).Decode(&msg)
+ if jerr == nil {
+ return Release{}, fmt.Errorf("unexpected status %v (%v) returned, message:\n %v", res.StatusCode, res.Status, msg.Message)
+ }
+ }
+
+ _ = res.Body.Close()
+ return Release{}, fmt.Errorf("unexpected status %v (%v) returned", res.StatusCode, res.Status)
+ }
+
+ buf, err := ioutil.ReadAll(res.Body)
+ if err != nil {
+ _ = res.Body.Close()
+ return Release{}, err
+ }
+
+ err = res.Body.Close()
+ if err != nil {
+ return Release{}, err
+ }
+
+ var release Release
+ err = json.Unmarshal(buf, &release)
+ if err != nil {
+ return Release{}, err
+ }
+
+ if release.TagName == "" {
+ return Release{}, errors.New("tag name for latest release is empty")
+ }
+
+ release.Version = release.TagName
+
+ return release, nil
+}
+
+func getGithubData(ctx context.Context, url string) ([]byte, error) {
+ req, err := http.NewRequest(http.MethodGet, url, nil)
+ if err != nil {
+ return nil, err
+ }
+
+ // request binary data
+ req.Header.Set("Accept", "application/octet-stream")
+
+ res, err := ctxhttp.Do(ctx, http.DefaultClient, req)
+ if err != nil {
+ return nil, err
+ }
+
+ if res.StatusCode != http.StatusOK {
+ return nil, fmt.Errorf("unexpected status %v (%v) returned", res.StatusCode, res.Status)
+ }
+
+ buf, err := ioutil.ReadAll(res.Body)
+ if err != nil {
+ _ = res.Body.Close()
+ return nil, err
+ }
+
+ err = res.Body.Close()
+ if err != nil {
+ return nil, err
+ }
+
+ return buf, nil
+}
+
+func getGithubDataFile(ctx context.Context, assets []Asset, suffix string) (filename string, data []byte, err error) {
+ var url string
+ for _, a := range assets {
+ if strings.HasSuffix(a.Name, suffix) {
+ url = a.URL
+ filename = a.Name
+ break
+ }
+ }
+
+ if url == "" {
+ return "", nil, fmt.Errorf("unable to find file with suffix %v", suffix)
+ }
+
+ glog.V(0).Infof("download %v\n", filename)
+ data, err = getGithubData(ctx, url)
+ if err != nil {
+ return "", nil, err
+ }
+
+ return filename, data, nil
+}
+
+func extractToFile(buf []byte, filename, target string) error {
+ var rd io.Reader = bytes.NewReader(buf)
+
+ switch filepath.Ext(filename) {
+ case ".gz":
+ gr, err := gzip.NewReader(rd)
+ if err != nil {
+ return err
+ }
+ defer gr.Close()
+ trd := tar.NewReader(gr)
+ hdr, terr := trd.Next()
+ if terr != nil {
+ glog.Errorf("uncompress file(%s) failed:%s", hdr.Name, terr)
+ return terr
+ }
+ rd = trd
+ case ".zip":
+ zrd, err := zip.NewReader(bytes.NewReader(buf), int64(len(buf)))
+ if err != nil {
+ return err
+ }
+
+ if len(zrd.File) != 1 {
+ return errors.New("ZIP archive contains more than one file")
+ }
+
+ file, err := zrd.File[0].Open()
+ if err != nil {
+ return err
+ }
+
+ defer func() {
+ _ = file.Close()
+ }()
+
+ rd = file
+ }
+
+ // Write everything to a temp file
+ dir := filepath.Dir(target)
+ new, err := ioutil.TempFile(dir, "weed")
+ if err != nil {
+ return err
+ }
+
+ n, err := io.Copy(new, rd)
+ if err != nil {
+ _ = new.Close()
+ _ = os.Remove(new.Name())
+ return err
+ }
+ if err = new.Sync(); err != nil {
+ return err
+ }
+ if err = new.Close(); err != nil {
+ return err
+ }
+
+ mode := os.FileMode(0755)
+ // attempt to find the original mode
+ if fi, err := os.Lstat(target); err == nil {
+ mode = fi.Mode()
+ }
+
+ // Remove the original binary.
+ if err := removeWeedBinary(dir, target); err != nil {
+ return err
+ }
+
+ // Rename the temp file to the final location atomically.
+ if err := os.Rename(new.Name(), target); err != nil {
+ return err
+ }
+
+ glog.V(0).Infof("saved %d bytes in %v\n", n, target)
+ return os.Chmod(target, mode)
+}
+
+// Rename (rather than remove) the running version. The running binary will be locked
+// on Windows and cannot be removed while still executing.
+func removeWeedBinary(dir, target string) error {
+ if runtime.GOOS == "linux" {
+ return nil
+ }
+ backup := filepath.Join(dir, filepath.Base(target)+".bak")
+ if _, err := os.Stat(backup); err == nil {
+ _ = os.Remove(backup)
+ }
+ if err := os.Rename(target, backup); err != nil {
+ return fmt.Errorf("unable to rename target file: %v", err)
+ }
+ return nil
+}
From 78fc9cdfa0cdd973d68be432cdcefb804ae5b723 Mon Sep 17 00:00:00 2001
From: guol-fnst <goul-fnst@fujitsu.com>
Date: Tue, 14 Jun 2022 19:10:54 +0800
Subject: [PATCH 13/38] add "copied from
https://github.com/restic/restic/tree/master/internal/selfupdate"
---
weed/command/command.go | 2 +-
weed/command/update.go | 6 ++++--
2 files changed, 5 insertions(+), 3 deletions(-)
diff --git a/weed/command/command.go b/weed/command/command.go
index b16de3eba..7635405dc 100644
--- a/weed/command/command.go
+++ b/weed/command/command.go
@@ -37,8 +37,8 @@ var Commands = []*Command{
cmdScaffold,
cmdServer,
cmdShell,
- cmdUpload,
cmdUpdate,
+ cmdUpload,
cmdVersion,
cmdVolume,
cmdWebDav,
diff --git a/weed/command/update.go b/weed/command/update.go
index bda904a85..64a664923 100644
--- a/weed/command/update.go
+++ b/weed/command/update.go
@@ -25,6 +25,8 @@ import (
"golang.org/x/net/context/ctxhttp"
)
+//copied from https://github.com/restic/restic/tree/master/internal/selfupdate
+
// Release collects data about a single release on GitHub.
type Release struct {
Name string `json:"name"`
@@ -59,8 +61,8 @@ func init() {
var cmdUpdate = &Command{
UsageLine: "update [-output=weed]",
- Short: "get latest stable version from https://github.com/chrislusf/seaweedfs",
- Long: `get latest stable version from https://github.com/chrislusf/seaweedfs`,
+ Short: "get latest version from https://github.com/chrislusf/seaweedfs",
+ Long: `get latest version from https://github.com/chrislusf/seaweedfs`,
}
func runUpdate(cmd *Command, args []string) bool {
From 14d82c3deaee2b7eecd95c5432c9a60b23a94033 Mon Sep 17 00:00:00 2001
From: "zhihao.qu" <zhihao.qu@ly.com>
Date: Tue, 14 Jun 2022 19:46:02 +0800
Subject: [PATCH 14/38] feat(filer.sync): add offset to path.
---
weed/command/filer_sync.go | 14 ++++++++++++--
1 file changed, 12 insertions(+), 2 deletions(-)
diff --git a/weed/command/filer_sync.go b/weed/command/filer_sync.go
index 7aa9c1e8d..dc7c569c3 100644
--- a/weed/command/filer_sync.go
+++ b/weed/command/filer_sync.go
@@ -182,7 +182,7 @@ func doSubscribeFilerMetaChanges(clientId int32, grpcDialOption grpc.DialOption,
// if first time, start from now
// if has previously synced, resume from that point of time
- sourceFilerOffsetTsNs, err := getOffset(grpcDialOption, targetFiler, SyncKeyPrefix, sourceFilerSignature)
+ sourceFilerOffsetTsNs, err := getOffset(grpcDialOption, targetFiler, getSignaturePrefixByPath(sourcePath), sourceFilerSignature)
if err != nil {
return err
}
@@ -214,7 +214,7 @@ func doSubscribeFilerMetaChanges(clientId int32, grpcDialOption grpc.DialOption,
now := time.Now().Nanosecond()
glog.V(0).Infof("sync %s to %s progressed to %v %0.2f/sec", sourceFiler, targetFiler, time.Unix(0, lastTsNs), float64(counter)/(float64(now-lastLogTsNs)/1e9))
lastLogTsNs = now
- return setOffset(grpcDialOption, targetFiler, SyncKeyPrefix, sourceFilerSignature, lastTsNs)
+ return setOffset(grpcDialOption, targetFiler, getSignaturePrefixByPath(sourcePath), sourceFilerSignature, lastTsNs)
})
return pb.FollowMetadata(sourceFiler, grpcDialOption, "syncTo_"+string(targetFiler), clientId,
@@ -226,6 +226,16 @@ const (
SyncKeyPrefix = "sync."
)
+// When each business is distinguished according to path, and offsets need to be maintained separately.
+func getSignaturePrefixByPath(path string) string {
+ // compatible historical version
+ if path == "/" {
+ return SyncKeyPrefix
+ } else {
+ return SyncKeyPrefix + path
+ }
+}
+
func getOffset(grpcDialOption grpc.DialOption, filer pb.ServerAddress, signaturePrefix string, signature int32) (lastOffsetTsNs int64, readErr error) {
readErr = pb.WithFilerClient(false, filer, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
From 28cda854eb1dc3fa648ff4950cabf6bfafe05902 Mon Sep 17 00:00:00 2001
From: "yulai.li" <blacktear23@gmail.com>
Date: Wed, 15 Jun 2022 00:30:49 +0800
Subject: [PATCH 15/38] Add filer command line parameter to let Filer UI
show/hide directory delete button
---
weed/command/filer.go | 3 ++
weed/server/filer_server.go | 3 +-
weed/server/filer_server_handlers_read_dir.go | 7 +++-
weed/server/filer_ui/filer.html | 40 +++++++++----------
4 files changed, 31 insertions(+), 22 deletions(-)
diff --git a/weed/command/filer.go b/weed/command/filer.go
index 2c91e6fec..e558b16aa 100644
--- a/weed/command/filer.go
+++ b/weed/command/filer.go
@@ -55,6 +55,7 @@ type FilerOptions struct {
debug *bool
debugPort *int
localSocket *string
+ showUIDirectoryDelete *bool
}
func init() {
@@ -82,6 +83,7 @@ func init() {
f.debug = cmdFiler.Flag.Bool("debug", false, "serves runtime profiling data, e.g., http://localhost:<debug.port>/debug/pprof/goroutine?debug=2")
f.debugPort = cmdFiler.Flag.Int("debug.port", 6060, "http port for debugging")
f.localSocket = cmdFiler.Flag.String("localSocket", "", "default to /tmp/seaweedfs-filer-<port>.sock")
+ f.showUIDirectoryDelete = cmdFiler.Flag.Bool("ui.deleteDir", false, "enable filer UI show delete directory button")
// start s3 on filer
filerStartS3 = cmdFiler.Flag.Bool("s3", false, "whether to start S3 gateway")
@@ -216,6 +218,7 @@ func (fo *FilerOptions) startFiler() {
Cipher: *fo.cipher,
SaveToFilerLimit: int64(*fo.saveToFilerLimit),
ConcurrentUploadLimit: int64(*fo.concurrentUploadLimitMB) * 1024 * 1024,
+ ShowUIDirectoryDelete: *fo.showUIDirectoryDelete,
})
if nfs_err != nil {
glog.Fatalf("Filer startup error: %v", nfs_err)
diff --git a/weed/server/filer_server.go b/weed/server/filer_server.go
index 8908b5e5f..6bf0261ee 100644
--- a/weed/server/filer_server.go
+++ b/weed/server/filer_server.go
@@ -3,7 +3,6 @@ package weed_server
import (
"context"
"fmt"
- "github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"net/http"
"os"
"sync"
@@ -17,6 +16,7 @@ import (
"github.com/chrislusf/seaweedfs/weed/operation"
"github.com/chrislusf/seaweedfs/weed/pb"
+ "github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
"github.com/chrislusf/seaweedfs/weed/util"
@@ -67,6 +67,7 @@ type FilerOption struct {
Cipher bool
SaveToFilerLimit int64
ConcurrentUploadLimit int64
+ ShowUIDirectoryDelete bool
}
type FilerServer struct {
diff --git a/weed/server/filer_server_handlers_read_dir.go b/weed/server/filer_server_handlers_read_dir.go
index 8382cfc76..eaf17fa18 100644
--- a/weed/server/filer_server_handlers_read_dir.go
+++ b/weed/server/filer_server_handlers_read_dir.go
@@ -73,7 +73,7 @@ func (fs *FilerServer) listDirectoryHandler(w http.ResponseWriter, r *http.Reque
return
}
- ui.StatusTpl.Execute(w, struct {
+ err = ui.StatusTpl.Execute(w, struct {
Path string
Breadcrumbs []ui.Breadcrumb
Entries interface{}
@@ -81,6 +81,7 @@ func (fs *FilerServer) listDirectoryHandler(w http.ResponseWriter, r *http.Reque
LastFileName string
ShouldDisplayLoadMore bool
EmptyFolder bool
+ ShowDirectoryDelete bool
}{
path,
ui.ToBreadcrumb(path),
@@ -89,5 +90,9 @@ func (fs *FilerServer) listDirectoryHandler(w http.ResponseWriter, r *http.Reque
lastFileName,
shouldDisplayLoadMore,
emptyFolder,
+ fs.option.ShowUIDirectoryDelete,
})
+ if err != nil {
+ glog.V(0).Infof("Template Execute Error: %v", err)
+ }
}
diff --git a/weed/server/filer_ui/filer.html b/weed/server/filer_ui/filer.html
index f9c35440e..c9d832e8f 100644
--- a/weed/server/filer_ui/filer.html
+++ b/weed/server/filer_ui/filer.html
@@ -109,38 +109,37 @@
<form class="upload-form">
<input type="file" id="fileElem" multiple onchange="handleFiles(this.files)">
- {{if .EmptyFolder}}
+ {{ if .EmptyFolder }}
<div class="row add-files">
+
</div>
- {{else}}
+ {{ else }}
<table width="100%" class="table table-hover">
- {{$path := .Path }}
+ {{ $path := .Path }}
+ {{ $showDirDel := .ShowDirectoryDelete }}
{{ range $entry_index, $entry := .Entries }}
<tr>
<td>
- {{if $entry.IsDirectory}}
+ {{ if $entry.IsDirectory }}
<span class="glyphicon glyphicon-folder-open" aria-hidden="true"></span>
<a href="{{ printpath $path "/" $entry.Name "/"}}" >
{{ $entry.Name }}
</a>
- {{else}}
+ {{ else }}
<a href="{{ printpath $path "/" $entry.Name }}" >
{{ $entry.Name }}
</a>
- {{end}}
+ {{ end }}
</td>
<td align="right" nowrap>
- {{if $entry.IsDirectory}}
- {{else}}
+ {{ if not $entry.IsDirectory }}
{{ $entry.Mime }}
- {{end}}
+ {{ end }}
</td>
<td align="right" nowrap>
- {{if $entry.IsDirectory}}
- {{else}}
+ {{ if not $entry.IsDirectory }}
{{ $entry.Size | humanizeBytes }}
- {{end}}
+ {{ end }}
</td>
<td align="right" nowrap>
{{ $entry.Timestamp.Format "2006-01-02 15:04" }}
@@ -150,31 +149,32 @@
<label class="btn" onclick="handleRename('{{ $entry.Name }}', '{{ printpath $path "/" }}')">
<span class="glyphicon glyphicon-edit" aria-hidden="true"></span>
</label>
- {{if $entry.IsDirectory}}
- <label class="btn" onclick="handleDelete('{{ printpath $path "/" $entry.Name "/" }}')">
+ {{ if and $entry.IsDirectory $showDirDel }}
+ <label class="btn" onclick="handleDelete('{{ printpath $path "/" $entry.Name "/" }}')">
<span class="glyphicon glyphicon-trash" aria-hidden="true"></span>
</label>
- {{else}}
+ {{ end }}
+ {{ if not $entry.IsDirectory }}
<label class="btn" onclick="handleDelete('{{ printpath $path "/" $entry.Name }}')">
<span class="glyphicon glyphicon-trash" aria-hidden="true"></span>
</label>
- {{end}}
+ {{ end }}
</div>
</td>
</tr>
{{ end }}
</table>
- {{end}}
+ {{ end }}
</form>
</div>
- {{if .ShouldDisplayLoadMore}}
+ {{ if .ShouldDisplayLoadMore }}
<div class="row">
- <a href={{ print .Path "?limit=" .Limit "&lastFileName=" .LastFileName}} >
+ <a href={{ print .Path "?limit=" .Limit "&lastFileName=" .LastFileName }} >
Load more
</a>
</div>
- {{end}}
+ {{ end }}
<br/>
<br/>
From 36d275697806678de4af19645a9110529ad13800 Mon Sep 17 00:00:00 2001
From: chrislu <chris.lu@gmail.com>
Date: Tue, 14 Jun 2022 11:16:47 -0700
Subject: [PATCH 16/38] volume: avoid writing too much data for range requests
in large files
related to https://github.com/chrislusf/seaweedfs/issues/3178
---
weed/storage/needle/needle_read_page.go | 7 ++++---
1 file changed, 4 insertions(+), 3 deletions(-)
diff --git a/weed/storage/needle/needle_read_page.go b/weed/storage/needle/needle_read_page.go
index c00195e93..e909869f3 100644
--- a/weed/storage/needle/needle_read_page.go
+++ b/weed/storage/needle/needle_read_page.go
@@ -14,9 +14,10 @@ func (n *Needle) ReadNeedleDataInto(r backend.BackendStorageFile, volumeOffset i
crc := CRC(0)
for x := needleOffset; x < needleOffset+size; x += int64(len(buf)) {
count, err := n.ReadNeedleData(r, volumeOffset, buf, x)
- if count > 0 {
- crc = crc.Update(buf[0:count])
- if _, err = writer.Write(buf[0:count]); err != nil {
+ toWrite := min(int64(count), needleOffset+size-x)
+ if toWrite > 0 {
+ crc = crc.Update(buf[0:toWrite])
+ if _, err = writer.Write(buf[0:toWrite]); err != nil {
return fmt.Errorf("ReadNeedleData write: %v", err)
}
}
From d4ef06cdcf320f8b8b17279586e0738894869eff Mon Sep 17 00:00:00 2001
From: chrislu <chris.lu@gmail.com>
Date: Tue, 14 Jun 2022 12:14:52 -0700
Subject: [PATCH 17/38] 3.11
---
k8s/helm_charts2/Chart.yaml | 4 ++--
weed/util/constants.go | 2 +-
2 files changed, 3 insertions(+), 3 deletions(-)
diff --git a/k8s/helm_charts2/Chart.yaml b/k8s/helm_charts2/Chart.yaml
index edb4d7f72..5e5fc59b4 100644
--- a/k8s/helm_charts2/Chart.yaml
+++ b/k8s/helm_charts2/Chart.yaml
@@ -1,5 +1,5 @@
apiVersion: v1
description: SeaweedFS
name: seaweedfs
-appVersion: "3.10"
-version: "3.10"
+appVersion: "3.11"
+version: "3.11"
diff --git a/weed/util/constants.go b/weed/util/constants.go
index 213eafae0..c0fea8b17 100644
--- a/weed/util/constants.go
+++ b/weed/util/constants.go
@@ -5,7 +5,7 @@ import (
)
var (
- VERSION_NUMBER = fmt.Sprintf("%.02f", 3.10)
+ VERSION_NUMBER = fmt.Sprintf("%.02f", 3.11)
VERSION = sizeLimit + " " + VERSION_NUMBER
COMMIT = ""
)
From 42d04c581b174d21ecbd137bac87256adeaa9fd3 Mon Sep 17 00:00:00 2001
From: "zhihao.qu" <zhihao.qu@ly.com>
Date: Wed, 15 Jun 2022 11:33:18 +0800
Subject: [PATCH 18/38] feat(filer.sync): add metricsServer in filer.sync.
Metrics include: (1) the offset of the filer.sync (2) the last send timestamp
of the filer subscription
---
weed/command/filer_sync.go | 11 ++++++++++-
weed/server/filer_grpc_server_sub_meta.go | 4 ++++
weed/stats/metrics.go | 18 ++++++++++++++++++
3 files changed, 32 insertions(+), 1 deletion(-)
diff --git a/weed/command/filer_sync.go b/weed/command/filer_sync.go
index dc7c569c3..dca4e693f 100644
--- a/weed/command/filer_sync.go
+++ b/weed/command/filer_sync.go
@@ -12,6 +12,7 @@ import (
"github.com/chrislusf/seaweedfs/weed/replication/sink/filersink"
"github.com/chrislusf/seaweedfs/weed/replication/source"
"github.com/chrislusf/seaweedfs/weed/security"
+ statsCollect "github.com/chrislusf/seaweedfs/weed/stats"
"github.com/chrislusf/seaweedfs/weed/util"
"github.com/chrislusf/seaweedfs/weed/util/grace"
"google.golang.org/grpc"
@@ -40,6 +41,7 @@ type SyncOptions struct {
bFromTsMs *int64
aProxyByFiler *bool
bProxyByFiler *bool
+ metricsHttpPort *int
clientId int32
}
@@ -72,6 +74,7 @@ func init() {
syncOptions.bFromTsMs = cmdFilerSynchronize.Flag.Int64("b.fromTsMs", 0, "synchronization from timestamp on filer B. The unit is millisecond")
syncCpuProfile = cmdFilerSynchronize.Flag.String("cpuprofile", "", "cpu profile output file")
syncMemProfile = cmdFilerSynchronize.Flag.String("memprofile", "", "memory profile output file")
+ syncOptions.metricsHttpPort = cmdFilerSynchronize.Flag.Int("metricsPort", 0, "metrics listen port")
syncOptions.clientId = util.RandomInt32()
}
@@ -103,6 +106,9 @@ func runFilerSynchronize(cmd *Command, args []string) bool {
filerA := pb.ServerAddress(*syncOptions.filerA)
filerB := pb.ServerAddress(*syncOptions.filerB)
+ // start filer.sync metrics server
+ go statsCollect.StartMetricsServer(*syncOptions.metricsHttpPort)
+
// read a filer signature
aFilerSignature, aFilerErr := replication.ReadFilerSignature(grpcDialOption, filerA)
if aFilerErr != nil {
@@ -210,14 +216,17 @@ func doSubscribeFilerMetaChanges(clientId int32, grpcDialOption grpc.DialOption,
}
var lastLogTsNs = time.Now().Nanosecond()
+ var clientName = fmt.Sprintf("syncTo_%s_From_%s", string(targetFiler), string(sourceFiler))
processEventFnWithOffset := pb.AddOffsetFunc(processEventFn, 3*time.Second, func(counter int64, lastTsNs int64) error {
now := time.Now().Nanosecond()
glog.V(0).Infof("sync %s to %s progressed to %v %0.2f/sec", sourceFiler, targetFiler, time.Unix(0, lastTsNs), float64(counter)/(float64(now-lastLogTsNs)/1e9))
lastLogTsNs = now
+ // collect synchronous offset
+ statsCollect.FilerSyncOffsetGauge.WithLabelValues(sourceFiler.String(), targetFiler.String(), clientName, sourcePath).Set(float64(lastTsNs))
return setOffset(grpcDialOption, targetFiler, getSignaturePrefixByPath(sourcePath), sourceFilerSignature, lastTsNs)
})
- return pb.FollowMetadata(sourceFiler, grpcDialOption, "syncTo_"+string(targetFiler), clientId,
+ return pb.FollowMetadata(sourceFiler, grpcDialOption, clientName, clientId,
sourcePath, nil, sourceFilerOffsetTsNs, 0, targetFilerSignature, processEventFnWithOffset, pb.RetryForeverOnError)
}
diff --git a/weed/server/filer_grpc_server_sub_meta.go b/weed/server/filer_grpc_server_sub_meta.go
index 0540400a3..da710234b 100644
--- a/weed/server/filer_grpc_server_sub_meta.go
+++ b/weed/server/filer_grpc_server_sub_meta.go
@@ -2,6 +2,7 @@ package weed_server
import (
"fmt"
+ "github.com/chrislusf/seaweedfs/weed/stats"
"strings"
"time"
@@ -229,6 +230,9 @@ func (fs *FilerServer) eachEventNotificationFn(req *filer_pb.SubscribeMetadataRe
}
}
+ // collect timestamps for path
+ stats.FilerServerLastSendTsOfSubscribeGauge.WithLabelValues(fs.option.Host.String(), req.ClientName, req.PathPrefix).Set(float64(tsNs))
+
message := &filer_pb.SubscribeMetadataResponse{
Directory: dirPath,
EventNotification: eventNotification,
diff --git a/weed/stats/metrics.go b/weed/stats/metrics.go
index 943aafff9..207b37c81 100644
--- a/weed/stats/metrics.go
+++ b/weed/stats/metrics.go
@@ -77,6 +77,14 @@ var (
Buckets: prometheus.ExponentialBuckets(0.0001, 2, 24),
}, []string{"type"})
+ FilerServerLastSendTsOfSubscribeGauge = prometheus.NewGaugeVec(
+ prometheus.GaugeOpts{
+ Namespace: "SeaweedFS",
+ Subsystem: "filer",
+ Name: "last_send_timestamp_of_subscribe",
+ Help: "The last send timestamp of the filer subscription.",
+ }, []string{"sourceFiler", "clientName", "path"})
+
FilerStoreCounter = prometheus.NewCounterVec(
prometheus.CounterOpts{
Namespace: "SeaweedFS",
@@ -94,6 +102,14 @@ var (
Buckets: prometheus.ExponentialBuckets(0.0001, 2, 24),
}, []string{"store", "type"})
+ FilerSyncOffsetGauge = prometheus.NewGaugeVec(
+ prometheus.GaugeOpts{
+ Namespace: "SeaweedFS",
+ Subsystem: "filerSync",
+ Name: "sync_offset",
+ Help: "The offset of the filer synchronization service.",
+ }, []string{"sourceFiler", "targetFiler", "clientName", "path"})
+
VolumeServerRequestCounter = prometheus.NewCounterVec(
prometheus.CounterOpts{
Namespace: "SeaweedFS",
@@ -179,6 +195,8 @@ func init() {
Gather.MustRegister(FilerRequestHistogram)
Gather.MustRegister(FilerStoreCounter)
Gather.MustRegister(FilerStoreHistogram)
+ Gather.MustRegister(FilerSyncOffsetGauge)
+ Gather.MustRegister(FilerServerLastSendTsOfSubscribeGauge)
Gather.MustRegister(collectors.NewGoCollector())
Gather.MustRegister(collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}))
From b90d213a87f94d22743000a977901fcb376ec07b Mon Sep 17 00:00:00 2001
From: "yulai.li" <blacktear23@gmail.com>
Date: Wed, 15 Jun 2022 11:37:37 +0800
Subject: [PATCH 19/38] Change ui.deleteDir default to true
---
weed/command/filer.go | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/weed/command/filer.go b/weed/command/filer.go
index e558b16aa..c9f9a1956 100644
--- a/weed/command/filer.go
+++ b/weed/command/filer.go
@@ -83,7 +83,7 @@ func init() {
f.debug = cmdFiler.Flag.Bool("debug", false, "serves runtime profiling data, e.g., http://localhost:<debug.port>/debug/pprof/goroutine?debug=2")
f.debugPort = cmdFiler.Flag.Int("debug.port", 6060, "http port for debugging")
f.localSocket = cmdFiler.Flag.String("localSocket", "", "default to /tmp/seaweedfs-filer-<port>.sock")
- f.showUIDirectoryDelete = cmdFiler.Flag.Bool("ui.deleteDir", false, "enable filer UI show delete directory button")
+ f.showUIDirectoryDelete = cmdFiler.Flag.Bool("ui.deleteDir", true, "enable filer UI show delete directory button")
// start s3 on filer
filerStartS3 = cmdFiler.Flag.Bool("s3", false, "whether to start S3 gateway")
From 4d0d1848c648c3e2e6596451fa20eabc468239dc Mon Sep 17 00:00:00 2001
From: "zhihao.qu" <zhihao.qu@ly.com>
Date: Wed, 15 Jun 2022 13:33:20 +0800
Subject: [PATCH 20/38] fix(filer.sync): modify clientName format : from -> to
---
weed/command/filer_sync.go | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/weed/command/filer_sync.go b/weed/command/filer_sync.go
index dca4e693f..b7da1baf9 100644
--- a/weed/command/filer_sync.go
+++ b/weed/command/filer_sync.go
@@ -216,7 +216,7 @@ func doSubscribeFilerMetaChanges(clientId int32, grpcDialOption grpc.DialOption,
}
var lastLogTsNs = time.Now().Nanosecond()
- var clientName = fmt.Sprintf("syncTo_%s_From_%s", string(targetFiler), string(sourceFiler))
+ var clientName = fmt.Sprintf("syncFrom_%s_To_%s", string(sourceFiler), string(targetFiler))
processEventFnWithOffset := pb.AddOffsetFunc(processEventFn, 3*time.Second, func(counter int64, lastTsNs int64) error {
now := time.Now().Nanosecond()
glog.V(0).Infof("sync %s to %s progressed to %v %0.2f/sec", sourceFiler, targetFiler, time.Unix(0, lastTsNs), float64(counter)/(float64(now-lastLogTsNs)/1e9))
From 695e8c75551b7de9f47723b682f0cb0ab4b74002 Mon Sep 17 00:00:00 2001
From: guol-fnst <goul-fnst@fujitsu.com>
Date: Wed, 15 Jun 2022 15:26:04 +0800
Subject: [PATCH 21/38] fix minor bug
---
weed/command/update.go | 44 +++++++++++++++---------------------------
1 file changed, 16 insertions(+), 28 deletions(-)
diff --git a/weed/command/update.go b/weed/command/update.go
index 64a664923..945ecc293 100644
--- a/weed/command/update.go
+++ b/weed/command/update.go
@@ -9,7 +9,6 @@ import (
"crypto/md5"
"encoding/hex"
"encoding/json"
- "errors"
"fmt"
"io"
"io/ioutil"
@@ -46,6 +45,13 @@ type Asset struct {
URL string `json:"url"`
}
+const githubAPITimeout = 30 * time.Second
+
+// githubError is returned by the GitHub API, e.g. for rate-limiting.
+type githubError struct {
+ Message string
+}
+
var (
updateOpt UpdateOptions
)
@@ -66,17 +72,6 @@ var cmdUpdate = &Command{
}
func runUpdate(cmd *Command, args []string) bool {
- weedPath := *updateOpt.Output
- if weedPath == "" {
- file, err := os.Executable()
- if err != nil {
- glog.Fatalf("unable to find executable:%s", err)
- return false
- }
-
- *updateOpt.Output = file
- }
-
fi, err := os.Lstat(*updateOpt.Output)
if err != nil {
dirname := filepath.Dir(*updateOpt.Output)
@@ -91,7 +86,7 @@ func runUpdate(cmd *Command, args []string) bool {
}
} else {
if !fi.Mode().IsRegular() {
- glog.Fatalf("output path %v is not a normal file, use --output to specify a different file path", updateOpt.Output)
+ glog.Fatalf("output path %v is not a normal file, use --output to specify a different file path", *updateOpt.Output)
return false
}
}
@@ -111,11 +106,6 @@ func runUpdate(cmd *Command, args []string) bool {
func downloadLatestStableRelease(ctx context.Context, target string) (version string, err error) {
currentVersion := util.VERSION_NUMBER
- largeDiskSuffix := ""
- if util.VolumeSizeLimitGB == 8000 {
- largeDiskSuffix = "_large_disk"
- }
-
rel, err := GitHubLatestRelease(ctx, "chrislusf", "seaweedfs")
if err != nil {
return "", err
@@ -128,6 +118,11 @@ func downloadLatestStableRelease(ctx context.Context, target string) (version st
glog.V(0).Infof("latest version is %v\n", rel.Version)
+ largeDiskSuffix := ""
+ if util.VolumeSizeLimitGB == 8000 {
+ largeDiskSuffix = "_large_disk"
+ }
+
ext := "tar.gz"
if runtime.GOOS == "windows" {
ext = "zip"
@@ -150,7 +145,7 @@ func downloadLatestStableRelease(ctx context.Context, target string) (version st
binaryMd5 := md5Ctx.Sum(nil)
if hex.EncodeToString(binaryMd5) != string(md5Val[0:32]) {
glog.Errorf("md5:'%s' '%s'", hex.EncodeToString(binaryMd5), string(md5Val[0:32]))
- err = errors.New("binary md5sum doesn't match")
+ err = fmt.Errorf("binary md5sum doesn't match")
return "", err
}
@@ -169,13 +164,6 @@ func (r Release) String() string {
len(r.Assets))
}
-const githubAPITimeout = 30 * time.Second
-
-// githubError is returned by the GitHub API, e.g. for rate-limiting.
-type githubError struct {
- Message string
-}
-
// GitHubLatestRelease uses the GitHub API to get information about the latest
// release of a repository.
func GitHubLatestRelease(ctx context.Context, owner, repo string) (Release, error) {
@@ -229,7 +217,7 @@ func GitHubLatestRelease(ctx context.Context, owner, repo string) (Release, erro
}
if release.TagName == "" {
- return Release{}, errors.New("tag name for latest release is empty")
+ return Release{}, fmt.Errorf("tag name for latest release is empty")
}
release.Version = release.TagName
@@ -316,7 +304,7 @@ func extractToFile(buf []byte, filename, target string) error {
}
if len(zrd.File) != 1 {
- return errors.New("ZIP archive contains more than one file")
+ return fmt.Errorf("ZIP archive contains more than one file")
}
file, err := zrd.File[0].Open()
From 358c0c0b37a2b6ce7eb136dc55dabef61ef4528d Mon Sep 17 00:00:00 2001
From: "yulai.li" <blacktear23@gmail.com>
Date: Wed, 15 Jun 2022 15:53:31 +0800
Subject: [PATCH 22/38] Fix: provide filer.ui.deleteDir options for server
command
---
weed/command/server.go | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/weed/command/server.go b/weed/command/server.go
index 4b6b6c642..ba71a44bd 100644
--- a/weed/command/server.go
+++ b/weed/command/server.go
@@ -2,8 +2,6 @@ package command
import (
"fmt"
- "github.com/chrislusf/seaweedfs/weed/pb"
- "github.com/chrislusf/seaweedfs/weed/util/grace"
"net/http"
"os"
"strings"
@@ -12,7 +10,9 @@ import (
stats_collect "github.com/chrislusf/seaweedfs/weed/stats"
"github.com/chrislusf/seaweedfs/weed/glog"
+ "github.com/chrislusf/seaweedfs/weed/pb"
"github.com/chrislusf/seaweedfs/weed/util"
+ "github.com/chrislusf/seaweedfs/weed/util/grace"
)
type ServerOptions struct {
@@ -114,6 +114,7 @@ func init() {
filerOptions.saveToFilerLimit = cmdServer.Flag.Int("filer.saveToFilerLimit", 0, "Small files smaller than this limit can be cached in filer store.")
filerOptions.concurrentUploadLimitMB = cmdServer.Flag.Int("filer.concurrentUploadLimitMB", 64, "limit total concurrent upload size")
filerOptions.localSocket = cmdServer.Flag.String("filer.localSocket", "", "default to /tmp/seaweedfs-filer-<port>.sock")
+ filerOptions.showUIDirectoryDelete = cmdServer.Flag.Bool("filer.ui.deleteDir", true, "enable filer UI show delete directory button")
serverOptions.v.port = cmdServer.Flag.Int("volume.port", 8080, "volume server http listen port")
serverOptions.v.portGrpc = cmdServer.Flag.Int("volume.port.grpc", 0, "volume server grpc listen port")
From 31a929f7f206b21cd2d62ad4d897820ebbaad50c Mon Sep 17 00:00:00 2001
From: Konstantin Lebedev <9497591+kmlebedev@users.noreply.github.com>
Date: Wed, 15 Jun 2022 15:36:43 +0500
Subject: [PATCH 23/38] logging processRangeRequest errors
---
weed/server/common.go | 4 ++++
1 file changed, 4 insertions(+)
diff --git a/weed/server/common.go b/weed/server/common.go
index 39a8637ac..f02ec67ac 100644
--- a/weed/server/common.go
+++ b/weed/server/common.go
@@ -284,6 +284,7 @@ func processRangeRequest(r *http.Request, w http.ResponseWriter, totalSize int64
if rangeReq == "" {
w.Header().Set("Content-Length", strconv.FormatInt(totalSize, 10))
if err := writeFn(bufferedWriter, 0, totalSize); err != nil {
+ glog.Errorf("processRangeRequest headers: %+v err: %v", w.Header(), err)
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
@@ -294,6 +295,7 @@ func processRangeRequest(r *http.Request, w http.ResponseWriter, totalSize int64
//mostly copy from src/pkg/net/http/fs.go
ranges, err := parseRange(rangeReq, totalSize)
if err != nil {
+ glog.Errorf("processRangeRequest headers: %+v err: %v", w.Header(), err)
http.Error(w, err.Error(), http.StatusRequestedRangeNotSatisfiable)
return
}
@@ -326,6 +328,7 @@ func processRangeRequest(r *http.Request, w http.ResponseWriter, totalSize int64
w.WriteHeader(http.StatusPartialContent)
err = writeFn(bufferedWriter, ra.start, ra.length)
if err != nil {
+ glog.Errorf("processRangeRequest headers: %+v err: %v", w.Header(), err)
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
@@ -365,6 +368,7 @@ func processRangeRequest(r *http.Request, w http.ResponseWriter, totalSize int64
}
w.WriteHeader(http.StatusPartialContent)
if _, err := io.CopyN(bufferedWriter, sendContent, sendSize); err != nil {
+ glog.Errorf("processRangeRequest err: %v", err)
http.Error(w, "Internal Error", http.StatusInternalServerError)
return
}
From 338705f375a37c0bb49aed2cea4d3f89d98ef70e Mon Sep 17 00:00:00 2001
From: ningfd <fangdi.ning@seetatech.com>
Date: Wed, 15 Jun 2022 19:20:13 +0800
Subject: [PATCH 24/38] fix(wdclient): GetLocations return
---
weed/wdclient/vid_map.go | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/weed/wdclient/vid_map.go b/weed/wdclient/vid_map.go
index cdd783d91..f7a9a0f1a 100644
--- a/weed/wdclient/vid_map.go
+++ b/weed/wdclient/vid_map.go
@@ -133,7 +133,7 @@ func (vc *vidMap) GetLocations(vid uint32) (locations []Location, found bool) {
return
}
locations, found = vc.ecVid2Locations[vid]
- return
+ return locations, found && len(locations) > 0
}
func (vc *vidMap) addLocation(vid uint32, location Location) {
From 3076ac101ead13d58e7b6c065af65437e3aff9cd Mon Sep 17 00:00:00 2001
From: liubaojiang <1838095916@qq.com>
Date: Thu, 16 Jun 2022 09:58:44 +0800
Subject: [PATCH 25/38] move vs.concurrentUploadLimit != 0 out of the lock
---
weed/server/volume_server_handlers.go | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/weed/server/volume_server_handlers.go b/weed/server/volume_server_handlers.go
index 7e4c11fed..ef37b48f5 100644
--- a/weed/server/volume_server_handlers.go
+++ b/weed/server/volume_server_handlers.go
@@ -59,10 +59,10 @@ func (vs *VolumeServer) privateStoreHandler(w http.ResponseWriter, r *http.Reque
contentLength := getContentLength(r)
// exclude the replication from the concurrentUploadLimitMB
- if r.URL.Query().Get("type") != "replicate" {
+ if r.URL.Query().Get("type") != "replicate" && vs.concurrentUploadLimit != 0 {
startTime := time.Now()
vs.inFlightUploadDataLimitCond.L.Lock()
- for vs.concurrentUploadLimit != 0 && vs.inFlightUploadDataSize > vs.concurrentUploadLimit {
+ for vs.inFlightUploadDataSize > vs.concurrentUploadLimit {
//wait timeout check
if startTime.Add(vs.inflightUploadDataTimeout).Before(time.Now()) {
vs.inFlightUploadDataLimitCond.L.Unlock()
From 1dbb925d298572afb09257ba3555374032799580 Mon Sep 17 00:00:00 2001
From: guol-fnst <goul-fnst@fujitsu.com>
Date: Wed, 15 Jun 2022 15:51:37 +0800
Subject: [PATCH 26/38] remove unused function
remove log when weed is up to date
add check whether weed is full version
---
weed/command/update.go | 26 ++++++++++++--------------
weed/command/update_full.go | 9 +++++++++
2 files changed, 21 insertions(+), 14 deletions(-)
create mode 100644 weed/command/update_full.go
diff --git a/weed/command/update.go b/weed/command/update.go
index 945ecc293..3f575d513 100644
--- a/weed/command/update.go
+++ b/weed/command/update.go
@@ -52,6 +52,9 @@ type githubError struct {
Message string
}
+//default version is not full version
+var isFullVersion = false
+
var (
updateOpt UpdateOptions
)
@@ -91,16 +94,11 @@ func runUpdate(cmd *Command, args []string) bool {
}
}
- glog.V(0).Infof("writing weed to %v\n", *updateOpt.Output)
-
v, err := downloadLatestStableRelease(context.Background(), *updateOpt.Output)
if err != nil {
- glog.Fatalf("unable to update weed: %v", err)
+ glog.Fatalf("unable to update weed to version %s: %v", err, v)
return false
}
-
- glog.V(0).Infof("successfully updated weed to version %v\n", v)
-
return true
}
@@ -123,12 +121,17 @@ func downloadLatestStableRelease(ctx context.Context, target string) (version st
largeDiskSuffix = "_large_disk"
}
+ fullSuffix := ""
+ if isFullVersion {
+ fullSuffix = "_full"
+ }
+
ext := "tar.gz"
if runtime.GOOS == "windows" {
ext = "zip"
}
- suffix := fmt.Sprintf("%s_%s%s.%s", runtime.GOOS, runtime.GOARCH, largeDiskSuffix, ext)
+ suffix := fmt.Sprintf("%s_%s%s%s.%s", runtime.GOOS, runtime.GOARCH, fullSuffix, largeDiskSuffix, ext)
md5Filename := fmt.Sprintf("%s.md5", suffix)
_, md5Val, err := getGithubDataFile(ctx, rel.Assets, md5Filename)
if err != nil {
@@ -152,18 +155,13 @@ func downloadLatestStableRelease(ctx context.Context, target string) (version st
err = extractToFile(buf, downloadFilename, target)
if err != nil {
return "", err
+ } else {
+ glog.V(0).Infof("successfully updated weed to version %v\n", rel.Version)
}
return rel.Version, nil
}
-func (r Release) String() string {
- return fmt.Sprintf("%v %v, %d assets",
- r.TagName,
- r.PublishedAt.Local().Format("2006-01-02 15:04:05"),
- len(r.Assets))
-}
-
// GitHubLatestRelease uses the GitHub API to get information about the latest
// release of a repository.
func GitHubLatestRelease(ctx context.Context, owner, repo string) (Release, error) {
diff --git a/weed/command/update_full.go b/weed/command/update_full.go
new file mode 100644
index 000000000..529f38219
--- /dev/null
+++ b/weed/command/update_full.go
@@ -0,0 +1,9 @@
+//go:build elastic && ydb && gocdk && hdfs
+// +build elastic,ydb,gocdk,hdfs
+
+package command
+
+//set true if gtags are set
+func init() {
+ isFullVersion = true
+}
From 4bc879264335ed0f96fc9604d261efe37554e470 Mon Sep 17 00:00:00 2001
From: guol-fnst <goul-fnst@fujitsu.com>
Date: Thu, 16 Jun 2022 11:19:49 +0800
Subject: [PATCH 27/38] support specific version of weed
---
weed/command/update.go | 55 +++++++++++++++++++++++++++---------------
1 file changed, 36 insertions(+), 19 deletions(-)
diff --git a/weed/command/update.go b/weed/command/update.go
index 3f575d513..2f25cbe32 100644
--- a/weed/command/update.go
+++ b/weed/command/update.go
@@ -60,18 +60,20 @@ var (
)
type UpdateOptions struct {
- Output *string
+ Output *string
+ Version *string
}
func init() {
- updateOpt.Output = cmdUpdate.Flag.String("output", "weed", "Save the latest weed as `filename`")
+ updateOpt.Output = cmdUpdate.Flag.String("output", "weed", "Save the weed as `filename` or /path/to/dir/filename.")
+ updateOpt.Version = cmdUpdate.Flag.String("version", "0", "The version of weed you want to download. If not specified, get the latest version.")
cmdUpdate.Run = runUpdate
}
var cmdUpdate = &Command{
UsageLine: "update [-output=weed]",
- Short: "get latest version from https://github.com/chrislusf/seaweedfs",
- Long: `get latest version from https://github.com/chrislusf/seaweedfs`,
+ Short: "get latest or specific version from https://github.com/chrislusf/seaweedfs",
+ Long: `get latest or specific version from https://github.com/chrislusf/seaweedfs`,
}
func runUpdate(cmd *Command, args []string) bool {
@@ -80,41 +82,45 @@ func runUpdate(cmd *Command, args []string) bool {
dirname := filepath.Dir(*updateOpt.Output)
di, err := os.Lstat(dirname)
if err != nil {
- glog.Fatalf("unable to find directory:%s", dirname)
+ glog.Errorf("unable to find directory:%s", dirname)
return false
}
if !di.Mode().IsDir() {
- glog.Fatalf("output parent path %v is not a directory, use --output to specify a different file path", dirname)
+ glog.Errorf("output parent path %v is not a directory, use --output to specify a different file path", dirname)
return false
}
} else {
if !fi.Mode().IsRegular() {
- glog.Fatalf("output path %v is not a normal file, use --output to specify a different file path", *updateOpt.Output)
+ glog.Errorf("output path %v is not a normal file, use --output to specify a different file path", *updateOpt.Output)
return false
}
}
- v, err := downloadLatestStableRelease(context.Background(), *updateOpt.Output)
+ _, err = downloadRelease(context.Background(), *updateOpt.Output, *updateOpt.Version)
if err != nil {
- glog.Fatalf("unable to update weed to version %s: %v", err, v)
+ glog.Errorf("unable to download weed: %v", err)
return false
}
return true
}
-func downloadLatestStableRelease(ctx context.Context, target string) (version string, err error) {
+func downloadRelease(ctx context.Context, target string, ver string) (version string, err error) {
currentVersion := util.VERSION_NUMBER
- rel, err := GitHubLatestRelease(ctx, "chrislusf", "seaweedfs")
+ rel, err := GitHubLatestRelease(ctx, ver, "chrislusf", "seaweedfs")
if err != nil {
return "", err
}
if rel.Version == currentVersion {
- glog.V(0).Infof("weed is up to date\n")
+ if ver == "0" {
+ glog.V(0).Infof("weed is up to date")
+ } else {
+ glog.V(0).Infof("no need to download the same version of weed ")
+ }
return currentVersion, nil
}
- glog.V(0).Infof("latest version is %v\n", rel.Version)
+ glog.V(0).Infof("download version: %s", rel.Version)
largeDiskSuffix := ""
if util.VolumeSizeLimitGB == 8000 {
@@ -162,13 +168,13 @@ func downloadLatestStableRelease(ctx context.Context, target string) (version st
return rel.Version, nil
}
-// GitHubLatestRelease uses the GitHub API to get information about the latest
+// GitHubLatestRelease uses the GitHub API to get information about the specific
// release of a repository.
-func GitHubLatestRelease(ctx context.Context, owner, repo string) (Release, error) {
+func GitHubLatestRelease(ctx context.Context, ver string, owner, repo string) (Release, error) {
ctx, cancel := context.WithTimeout(ctx, githubAPITimeout)
defer cancel()
- url := fmt.Sprintf("https://api.github.com/repos/%s/%s/releases/latest", owner, repo)
+ url := fmt.Sprintf("https://api.github.com/repos/%s/%s/releases", owner, repo)
req, err := http.NewRequest(http.MethodGet, url, nil)
if err != nil {
return Release{}, err
@@ -209,17 +215,28 @@ func GitHubLatestRelease(ctx context.Context, owner, repo string) (Release, erro
}
var release Release
- err = json.Unmarshal(buf, &release)
+ var releaseList []Release
+ err = json.Unmarshal(buf, &releaseList)
if err != nil {
return Release{}, err
}
+ if ver == "0" {
+ release = releaseList[0]
+ glog.V(0).Infof("latest version is %v\n", release.TagName)
+ } else {
+ for _, r := range releaseList {
+ if r.TagName == ver {
+ release = r
+ break
+ }
+ }
+ }
if release.TagName == "" {
- return Release{}, fmt.Errorf("tag name for latest release is empty")
+ return Release{}, fmt.Errorf("can not find the specific version")
}
release.Version = release.TagName
-
return release, nil
}
From 62f74f5e3cce0668478c179f81fd1a9c5916e455 Mon Sep 17 00:00:00 2001
From: zzq09494 <zzq09494@ly.com>
Date: Thu, 16 Jun 2022 13:21:25 +0800
Subject: [PATCH 28/38] add bucket label to s3 prometheus metrics
---
weed/s3api/stats.go | 6 ++++--
weed/stats/metrics.go | 5 +++--
2 files changed, 7 insertions(+), 4 deletions(-)
diff --git a/weed/s3api/stats.go b/weed/s3api/stats.go
index 973d8c0eb..003807a25 100644
--- a/weed/s3api/stats.go
+++ b/weed/s3api/stats.go
@@ -1,6 +1,7 @@
package s3api
import (
+ "github.com/chrislusf/seaweedfs/weed/s3api/s3_constants"
stats_collect "github.com/chrislusf/seaweedfs/weed/stats"
"net/http"
"strconv"
@@ -27,11 +28,12 @@ func (r *StatusRecorder) Flush() {
func track(f http.HandlerFunc, action string) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
+ bucket, _ := s3_constants.GetBucketAndObject(r)
w.Header().Set("Server", "SeaweedFS S3")
recorder := NewStatusResponseWriter(w)
start := time.Now()
f(recorder, r)
- stats_collect.S3RequestHistogram.WithLabelValues(action).Observe(time.Since(start).Seconds())
- stats_collect.S3RequestCounter.WithLabelValues(action, strconv.Itoa(recorder.Status)).Inc()
+ stats_collect.S3RequestHistogram.WithLabelValues(action, bucket).Observe(time.Since(start).Seconds())
+ stats_collect.S3RequestCounter.WithLabelValues(action, strconv.Itoa(recorder.Status), bucket).Inc()
}
}
diff --git a/weed/stats/metrics.go b/weed/stats/metrics.go
index 207b37c81..f0b810608 100644
--- a/weed/stats/metrics.go
+++ b/weed/stats/metrics.go
@@ -173,7 +173,8 @@ var (
Subsystem: "s3",
Name: "request_total",
Help: "Counter of s3 requests.",
- }, []string{"type", "code"})
+ }, []string{"type", "code", "bucket"})
+
S3RequestHistogram = prometheus.NewHistogramVec(
prometheus.HistogramOpts{
Namespace: "SeaweedFS",
@@ -181,7 +182,7 @@ var (
Name: "request_seconds",
Help: "Bucketed histogram of s3 request processing time.",
Buckets: prometheus.ExponentialBuckets(0.0001, 2, 24),
- }, []string{"type"})
+ }, []string{"type", "bucket"})
)
func init() {
From 0a613876caa22542cac3bd99c19f44e64846a52f Mon Sep 17 00:00:00 2001
From: zzq09494 <zzq09494@ly.com>
Date: Thu, 16 Jun 2022 13:50:16 +0800
Subject: [PATCH 29/38] add bucket label to the grafana dashboard
---
other/metrics/grafana_seaweedfs.json | 16 +++++++++-------
1 file changed, 9 insertions(+), 7 deletions(-)
diff --git a/other/metrics/grafana_seaweedfs.json b/other/metrics/grafana_seaweedfs.json
index 3b9b222b4..d10fb7c51 100644
--- a/other/metrics/grafana_seaweedfs.json
+++ b/other/metrics/grafana_seaweedfs.json
@@ -539,11 +539,12 @@
"step": 60
},
{
- "expr": "histogram_quantile(0.90, sum(rate(SeaweedFS_s3_request_seconds_bucket[1m])) by (le, type))",
+ "expr": "histogram_quantile(0.90, sum(rate(SeaweedFS_s3_request_seconds_bucket[1m])) by (le, type, bucket))",
"format": "time_series",
"hide": false,
+ "interval": "",
"intervalFactor": 2,
- "legendFormat": "{{type}}",
+ "legendFormat": "{{bucket}} {{type}}",
"refId": "B",
"step": 60
}
@@ -645,11 +646,12 @@
"step": 60
},
{
- "expr": "histogram_quantile(0.95, sum(rate(SeaweedFS_s3_request_seconds_bucket[1m])) by (le, type))",
+ "expr": "histogram_quantile(0.95, sum(rate(SeaweedFS_s3_request_seconds_bucket[1m])) by (le, type,bucket))",
"format": "time_series",
"hide": false,
+ "interval": "",
"intervalFactor": 2,
- "legendFormat": "{{type}}",
+ "legendFormat": "{{bucket}} {{type}}",
"refId": "B",
"step": 60
}
@@ -751,11 +753,11 @@
"step": 60
},
{
- "expr": "histogram_quantile(0.99, sum(rate(SeaweedFS_s3_request_seconds_bucket[1m])) by (le, type))",
+ "expr": "histogram_quantile(0.99, sum(rate(SeaweedFS_s3_request_seconds_bucket[1m])) by (le, type, bucket))",
"format": "time_series",
"hide": false,
"intervalFactor": 2,
- "legendFormat": "{{type}}",
+ "legendFormat": "{{bucket}} {{type}}",
"refId": "B",
"step": 60
}
@@ -864,7 +866,7 @@
"expr": "rate(SeaweedFS_s3_request_total[1m])",
"format": "time_series",
"intervalFactor": 2,
- "legendFormat": "{{type}}",
+ "legendFormat": "{{bucket}} {{type}}",
"refId": "A",
"step": 30
}
From 6449114e5e88953b0fe22b50b46d91a7bed52523 Mon Sep 17 00:00:00 2001
From: zzq09494 <zzq09494@ly.com>
Date: Thu, 16 Jun 2022 13:52:36 +0800
Subject: [PATCH 30/38] format
---
other/metrics/grafana_seaweedfs.json | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/other/metrics/grafana_seaweedfs.json b/other/metrics/grafana_seaweedfs.json
index d10fb7c51..88844b3c3 100644
--- a/other/metrics/grafana_seaweedfs.json
+++ b/other/metrics/grafana_seaweedfs.json
@@ -646,7 +646,7 @@
"step": 60
},
{
- "expr": "histogram_quantile(0.95, sum(rate(SeaweedFS_s3_request_seconds_bucket[1m])) by (le, type,bucket))",
+ "expr": "histogram_quantile(0.95, sum(rate(SeaweedFS_s3_request_seconds_bucket[1m])) by (le, type, bucket))",
"format": "time_series",
"hide": false,
"interval": "",
From df0ce31a2ee87bf4550da20cf7c1095d154b24e3 Mon Sep 17 00:00:00 2001
From: liubaojiang <1838095916@qq.com>
Date: Thu, 16 Jun 2022 14:07:11 +0800
Subject: [PATCH 31/38] add condition when inFlightUploadDataLimitCond signal
---
weed/server/volume_server_handlers.go | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/weed/server/volume_server_handlers.go b/weed/server/volume_server_handlers.go
index ef37b48f5..293f36f14 100644
--- a/weed/server/volume_server_handlers.go
+++ b/weed/server/volume_server_handlers.go
@@ -79,7 +79,9 @@ func (vs *VolumeServer) privateStoreHandler(w http.ResponseWriter, r *http.Reque
atomic.AddInt64(&vs.inFlightUploadDataSize, contentLength)
defer func() {
atomic.AddInt64(&vs.inFlightUploadDataSize, -contentLength)
- vs.inFlightUploadDataLimitCond.Signal()
+ if vs.concurrentUploadLimit != 0 {
+ vs.inFlightUploadDataLimitCond.Signal()
+ }
}()
// processs uploads
From e024586ff11d025ce2dd27822a2b04e01260fb6c Mon Sep 17 00:00:00 2001
From: chrislu <chris.lu@gmail.com>
Date: Wed, 15 Jun 2022 23:19:39 -0700
Subject: [PATCH 32/38] adjust conditions
---
weed/server/volume_server_handlers.go | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/weed/server/volume_server_handlers.go b/weed/server/volume_server_handlers.go
index 293f36f14..f219dc9b9 100644
--- a/weed/server/volume_server_handlers.go
+++ b/weed/server/volume_server_handlers.go
@@ -59,7 +59,8 @@ func (vs *VolumeServer) privateStoreHandler(w http.ResponseWriter, r *http.Reque
contentLength := getContentLength(r)
// exclude the replication from the concurrentUploadLimitMB
- if r.URL.Query().Get("type") != "replicate" && vs.concurrentUploadLimit != 0 {
+ shouldWatchUploadLimit := r.URL.Query().Get("type") != "replicate" && vs.concurrentUploadLimit != 0
+ if shouldWatchUploadLimit {
startTime := time.Now()
vs.inFlightUploadDataLimitCond.L.Lock()
for vs.inFlightUploadDataSize > vs.concurrentUploadLimit {
@@ -79,7 +80,7 @@ func (vs *VolumeServer) privateStoreHandler(w http.ResponseWriter, r *http.Reque
atomic.AddInt64(&vs.inFlightUploadDataSize, contentLength)
defer func() {
atomic.AddInt64(&vs.inFlightUploadDataSize, -contentLength)
- if vs.concurrentUploadLimit != 0 {
+ if shouldWatchUploadLimit {
vs.inFlightUploadDataLimitCond.Signal()
}
}()
From 995e221e97d2cd9982d07c699afec4815d45e557 Mon Sep 17 00:00:00 2001
From: guol-fnst <goul-fnst@fujitsu.com>
Date: Thu, 16 Jun 2022 15:56:34 +0800
Subject: [PATCH 33/38] remove -output, add -dir,-name for more flexibility
---
weed/command/update.go | 64 ++++++++++++++++++------------------------
1 file changed, 27 insertions(+), 37 deletions(-)
diff --git a/weed/command/update.go b/weed/command/update.go
index 2f25cbe32..df5988b6e 100644
--- a/weed/command/update.go
+++ b/weed/command/update.go
@@ -60,13 +60,17 @@ var (
)
type UpdateOptions struct {
- Output *string
+ dir *string
+ name *string
Version *string
}
func init() {
- updateOpt.Output = cmdUpdate.Flag.String("output", "weed", "Save the weed as `filename` or /path/to/dir/filename.")
- updateOpt.Version = cmdUpdate.Flag.String("version", "0", "The version of weed you want to download. If not specified, get the latest version.")
+ path, _ := os.Executable()
+ _, name := filepath.Split(path)
+ updateOpt.dir = cmdUpdate.Flag.String("dir", filepath.Dir(path), "directory to save new weed.")
+ updateOpt.name = cmdUpdate.Flag.String("name", name, "name of new weed.On windows, name shouldn't be same to the orignial name.")
+ updateOpt.Version = cmdUpdate.Flag.String("version", "0", "specific version of weed you want to download. If not specified, get the latest version.")
cmdUpdate.Run = runUpdate
}
@@ -77,26 +81,33 @@ var cmdUpdate = &Command{
}
func runUpdate(cmd *Command, args []string) bool {
- fi, err := os.Lstat(*updateOpt.Output)
- if err != nil {
- dirname := filepath.Dir(*updateOpt.Output)
- di, err := os.Lstat(dirname)
- if err != nil {
- glog.Errorf("unable to find directory:%s", dirname)
- return false
- }
- if !di.Mode().IsDir() {
- glog.Errorf("output parent path %v is not a directory, use --output to specify a different file path", dirname)
+ path, _ := os.Executable()
+ _, name := filepath.Split(path)
+
+ if *updateOpt.dir != "" {
+ if err := util.TestFolderWritable(util.ResolvePath(*updateOpt.dir)); err != nil {
+ glog.Fatalf("Check Folder(-dir) Writable %s : %s", *updateOpt.dir, err)
return false
}
} else {
- if !fi.Mode().IsRegular() {
- glog.Errorf("output path %v is not a normal file, use --output to specify a different file path", *updateOpt.Output)
+ *updateOpt.dir = filepath.Dir(path)
+ }
+
+ if *updateOpt.name == "" {
+ *updateOpt.name = name
+ }
+
+ if runtime.GOOS == "windows" {
+ if *updateOpt.name == name || *updateOpt.name == "" {
+ glog.Fatalf("On windows, name of the new weed shouldn't be same to the orignial name.")
return false
}
}
- _, err = downloadRelease(context.Background(), *updateOpt.Output, *updateOpt.Version)
+ target := *updateOpt.dir + "/" + *updateOpt.name
+ glog.V(0).Infof("new weed will be saved to %s", target)
+
+ _, err := downloadRelease(context.Background(), target, *updateOpt.Version)
if err != nil {
glog.Errorf("unable to download weed: %v", err)
return false
@@ -360,11 +371,6 @@ func extractToFile(buf []byte, filename, target string) error {
mode = fi.Mode()
}
- // Remove the original binary.
- if err := removeWeedBinary(dir, target); err != nil {
- return err
- }
-
// Rename the temp file to the final location atomically.
if err := os.Rename(new.Name(), target); err != nil {
return err
@@ -373,19 +379,3 @@ func extractToFile(buf []byte, filename, target string) error {
glog.V(0).Infof("saved %d bytes in %v\n", n, target)
return os.Chmod(target, mode)
}
-
-// Rename (rather than remove) the running version. The running binary will be locked
-// on Windows and cannot be removed while still executing.
-func removeWeedBinary(dir, target string) error {
- if runtime.GOOS == "linux" {
- return nil
- }
- backup := filepath.Join(dir, filepath.Base(target)+".bak")
- if _, err := os.Stat(backup); err == nil {
- _ = os.Remove(backup)
- }
- if err := os.Rename(target, backup); err != nil {
- return fmt.Errorf("unable to rename target file: %v", err)
- }
- return nil
-}
From aaa9938353a02ca38cfe4a309b857c1ff8f738ca Mon Sep 17 00:00:00 2001
From: chrislu <chris.lu@gmail.com>
Date: Thu, 16 Jun 2022 01:26:36 -0700
Subject: [PATCH 34/38] Revert "adjust conditions"
This reverts commit e024586ff11d025ce2dd27822a2b04e01260fb6c.
---
weed/server/volume_server_handlers.go | 5 ++---
1 file changed, 2 insertions(+), 3 deletions(-)
diff --git a/weed/server/volume_server_handlers.go b/weed/server/volume_server_handlers.go
index f219dc9b9..293f36f14 100644
--- a/weed/server/volume_server_handlers.go
+++ b/weed/server/volume_server_handlers.go
@@ -59,8 +59,7 @@ func (vs *VolumeServer) privateStoreHandler(w http.ResponseWriter, r *http.Reque
contentLength := getContentLength(r)
// exclude the replication from the concurrentUploadLimitMB
- shouldWatchUploadLimit := r.URL.Query().Get("type") != "replicate" && vs.concurrentUploadLimit != 0
- if shouldWatchUploadLimit {
+ if r.URL.Query().Get("type") != "replicate" && vs.concurrentUploadLimit != 0 {
startTime := time.Now()
vs.inFlightUploadDataLimitCond.L.Lock()
for vs.inFlightUploadDataSize > vs.concurrentUploadLimit {
@@ -80,7 +79,7 @@ func (vs *VolumeServer) privateStoreHandler(w http.ResponseWriter, r *http.Reque
atomic.AddInt64(&vs.inFlightUploadDataSize, contentLength)
defer func() {
atomic.AddInt64(&vs.inFlightUploadDataSize, -contentLength)
- if shouldWatchUploadLimit {
+ if vs.concurrentUploadLimit != 0 {
vs.inFlightUploadDataLimitCond.Signal()
}
}()
From cd078471c4a3f588d452d1ae92b7489a5186957b Mon Sep 17 00:00:00 2001
From: guol-fnst <goul-fnst@fujitsu.com>
Date: Thu, 16 Jun 2022 16:52:57 +0800
Subject: [PATCH 35/38] fix path bug on windows fix usage
---
weed/command/update.go | 9 +++++----
1 file changed, 5 insertions(+), 4 deletions(-)
diff --git a/weed/command/update.go b/weed/command/update.go
index df5988b6e..8e0a76016 100644
--- a/weed/command/update.go
+++ b/weed/command/update.go
@@ -69,13 +69,13 @@ func init() {
path, _ := os.Executable()
_, name := filepath.Split(path)
updateOpt.dir = cmdUpdate.Flag.String("dir", filepath.Dir(path), "directory to save new weed.")
- updateOpt.name = cmdUpdate.Flag.String("name", name, "name of new weed.On windows, name shouldn't be same to the orignial name.")
+ updateOpt.name = cmdUpdate.Flag.String("name", name, "name of new weed. On windows, name shouldn't be same to the orignial name.")
updateOpt.Version = cmdUpdate.Flag.String("version", "0", "specific version of weed you want to download. If not specified, get the latest version.")
cmdUpdate.Run = runUpdate
}
var cmdUpdate = &Command{
- UsageLine: "update [-output=weed]",
+ UsageLine: "weed update -dir=/path/to/dir -name=name -version=x.xx",
Short: "get latest or specific version from https://github.com/chrislusf/seaweedfs",
Long: `get latest or specific version from https://github.com/chrislusf/seaweedfs`,
}
@@ -97,14 +97,15 @@ func runUpdate(cmd *Command, args []string) bool {
*updateOpt.name = name
}
+ target := filepath.Join(*updateOpt.dir, *updateOpt.name)
+
if runtime.GOOS == "windows" {
- if *updateOpt.name == name || *updateOpt.name == "" {
+ if target == path {
glog.Fatalf("On windows, name of the new weed shouldn't be same to the orignial name.")
return false
}
}
- target := *updateOpt.dir + "/" + *updateOpt.name
glog.V(0).Infof("new weed will be saved to %s", target)
_, err := downloadRelease(context.Background(), target, *updateOpt.Version)
From 625fd16a2e5eb676e8c9eaa0268f78246c69b1ef Mon Sep 17 00:00:00 2001
From: chrislu <chris.lu@gmail.com>
Date: Thu, 16 Jun 2022 15:39:29 -0700
Subject: [PATCH 36/38] reduce upfront memory usage for low density volume
---
weed/storage/needle_map/compact_map.go | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/weed/storage/needle_map/compact_map.go b/weed/storage/needle_map/compact_map.go
index 3d2047f99..ccce8f108 100644
--- a/weed/storage/needle_map/compact_map.go
+++ b/weed/storage/needle_map/compact_map.go
@@ -8,7 +8,7 @@ import (
)
const (
- batch = 100000
+ batch = 10000
)
type SectionalNeedleId uint32
From c7c20881f5543e9ab2f413d32ad2943c3d1c4828 Mon Sep 17 00:00:00 2001
From: guol-fnst <goul-fnst@fujitsu.com>
Date: Fri, 17 Jun 2022 12:40:54 +0800
Subject: [PATCH 37/38] remove "weed" from UsageLine, or weed will not show
usage of weed update
---
weed/command/update.go | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/weed/command/update.go b/weed/command/update.go
index 8e0a76016..2d0dc42ad 100644
--- a/weed/command/update.go
+++ b/weed/command/update.go
@@ -75,7 +75,7 @@ func init() {
}
var cmdUpdate = &Command{
- UsageLine: "weed update -dir=/path/to/dir -name=name -version=x.xx",
+ UsageLine: "update [-dir=/path/to/dir] [-name=name] [-version=x.xx]",
Short: "get latest or specific version from https://github.com/chrislusf/seaweedfs",
Long: `get latest or specific version from https://github.com/chrislusf/seaweedfs`,
}
From fdacd94af57a0b9040a396154e7c322bc56dd16e Mon Sep 17 00:00:00 2001
From: geekboood <booodpoooq20@gmail.com>
Date: Sun, 19 Jun 2022 16:54:04 +0800
Subject: [PATCH 38/38] fix: invalid chunk data when failed to read manifests
---
weed/filer/filechunk_manifest.go | 7 ++++---
weed/filer/filechunks.go | 8 ++++++--
2 files changed, 10 insertions(+), 5 deletions(-)
diff --git a/weed/filer/filechunk_manifest.go b/weed/filer/filechunk_manifest.go
index 091bbee5a..4eb657dfa 100644
--- a/weed/filer/filechunk_manifest.go
+++ b/weed/filer/filechunk_manifest.go
@@ -3,7 +3,6 @@ package filer
import (
"bytes"
"fmt"
- "github.com/chrislusf/seaweedfs/weed/wdclient"
"io"
"math"
"net/url"
@@ -11,6 +10,8 @@ import (
"sync"
"time"
+ "github.com/chrislusf/seaweedfs/weed/wdclient"
+
"github.com/golang/protobuf/proto"
"github.com/chrislusf/seaweedfs/weed/glog"
@@ -63,14 +64,14 @@ func ResolveChunkManifest(lookupFileIdFn wdclient.LookupFileIdFunctionType, chun
resolvedChunks, err := ResolveOneChunkManifest(lookupFileIdFn, chunk)
if err != nil {
- return chunks, nil, err
+ return dataChunks, nil, err
}
manifestChunks = append(manifestChunks, chunk)
// recursive
subDataChunks, subManifestChunks, subErr := ResolveChunkManifest(lookupFileIdFn, resolvedChunks, startOffset, stopOffset)
if subErr != nil {
- return chunks, nil, subErr
+ return dataChunks, nil, subErr
}
dataChunks = append(dataChunks, subDataChunks...)
manifestChunks = append(manifestChunks, subManifestChunks...)
diff --git a/weed/filer/filechunks.go b/weed/filer/filechunks.go
index 208ef8095..48b344bf8 100644
--- a/weed/filer/filechunks.go
+++ b/weed/filer/filechunks.go
@@ -3,11 +3,12 @@ package filer
import (
"bytes"
"fmt"
- "github.com/chrislusf/seaweedfs/weed/wdclient"
- "golang.org/x/exp/slices"
"math"
"sync"
+ "github.com/chrislusf/seaweedfs/weed/wdclient"
+ "golang.org/x/exp/slices"
+
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"github.com/chrislusf/seaweedfs/weed/util"
)
@@ -248,6 +249,9 @@ func MergeIntoVisibles(visibles []VisibleInterval, chunk *filer_pb.FileChunk) (n
func NonOverlappingVisibleIntervals(lookupFileIdFn wdclient.LookupFileIdFunctionType, chunks []*filer_pb.FileChunk, startOffset int64, stopOffset int64) (visibles []VisibleInterval, err error) {
chunks, _, err = ResolveChunkManifest(lookupFileIdFn, chunks, startOffset, stopOffset)
+ if err != nil {
+ return
+ }
visibles2 := readResolvedChunks(chunks)