From 8a40fa8993b531f04d3b7100679ff5675d3f8c90 Mon Sep 17 00:00:00 2001 From: chrislu Date: Sat, 17 Dec 2022 13:18:35 -0800 Subject: [PATCH 01/25] more detailed logs --- weed/command/filer_sync.go | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/weed/command/filer_sync.go b/weed/command/filer_sync.go index 00afab535..992b9dd4e 100644 --- a/weed/command/filer_sync.go +++ b/weed/command/filer_sync.go @@ -399,7 +399,11 @@ func genProcessFunction(sourcePath string, targetPath string, excludePaths []str return nil } key := buildKey(dataSink, message, targetPath, sourceNewKey, sourcePath) - return dataSink.CreateEntry(key, message.NewEntry, message.Signatures) + if err := dataSink.CreateEntry(key, message.NewEntry, message.Signatures); err != nil { + return fmt.Errorf("create entry1 : %v", err) + } else { + return nil + } } // this is something special? @@ -427,7 +431,11 @@ func genProcessFunction(sourcePath string, targetPath string, excludePaths []str } // create the new entry newKey := buildKey(dataSink, message, targetPath, sourceNewKey, sourcePath) - return dataSink.CreateEntry(newKey, message.NewEntry, message.Signatures) + if err := dataSink.CreateEntry(newKey, message.NewEntry, message.Signatures); err != nil { + return fmt.Errorf("create entry2 : %v", err) + } else { + return nil + } } else { // new key is outside of the watched directory @@ -441,7 +449,11 @@ func genProcessFunction(sourcePath string, targetPath string, excludePaths []str if strings.HasPrefix(string(sourceNewKey), sourcePath) { // new key is in the watched directory key := buildKey(dataSink, message, targetPath, sourceNewKey, sourcePath) - return dataSink.CreateEntry(key, message.NewEntry, message.Signatures) + if err := dataSink.CreateEntry(key, message.NewEntry, message.Signatures); err != nil { + return fmt.Errorf("create entry3 : %v", err) + } else { + return nil + } } else { // new key is also outside of the watched directory // skip From 038bab372607822bb3000092ac8a247c00a94ccd Mon Sep 17 00:00:00 2001 From: lfhy <821923589@qq.com> Date: Sun, 18 Dec 2022 12:07:02 +0800 Subject: [PATCH 02/25] add -filer.path to webdav command (#4061) --- weed/command/webdav.go | 5 ++++- weed/server/webdav_server.go | 12 ++++++++++-- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/weed/command/webdav.go b/weed/command/webdav.go index ea19ad1a7..987fc388e 100644 --- a/weed/command/webdav.go +++ b/weed/command/webdav.go @@ -13,7 +13,7 @@ import ( "github.com/seaweedfs/seaweedfs/weed/pb" "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" "github.com/seaweedfs/seaweedfs/weed/security" - "github.com/seaweedfs/seaweedfs/weed/server" + weed_server "github.com/seaweedfs/seaweedfs/weed/server" "github.com/seaweedfs/seaweedfs/weed/util" ) @@ -23,6 +23,7 @@ var ( type WebDavOption struct { filer *string + filerRootPath *string port *int collection *string replication *string @@ -44,6 +45,7 @@ func init() { webDavStandaloneOptions.tlsCertificate = cmdWebDav.Flag.String("cert.file", "", "path to the TLS certificate file") webDavStandaloneOptions.cacheDir = cmdWebDav.Flag.String("cacheDir", os.TempDir(), "local cache directory for file chunks") webDavStandaloneOptions.cacheSizeMB = cmdWebDav.Flag.Int64("cacheCapacityMB", 0, "local cache capacity in MB") + webDavStandaloneOptions.filerRootPath = cmdWebDav.Flag.String("filer.path", "/", "use this remote path from filer server") } var cmdWebDav = &Command{ @@ -104,6 +106,7 @@ func (wo *WebDavOption) startWebDav() bool { ws, webdavServer_err := weed_server.NewWebDavServer(&weed_server.WebDavOption{ Filer: filerAddress, + FilerRootPath: *wo.filerRootPath, GrpcDialOption: grpcDialOption, Collection: *wo.collection, Replication: *wo.replication, diff --git a/weed/server/webdav_server.go b/weed/server/webdav_server.go index 58c47671f..80b882181 100644 --- a/weed/server/webdav_server.go +++ b/weed/server/webdav_server.go @@ -26,6 +26,7 @@ import ( type WebDavOption struct { Filer pb.ServerAddress + FilerRootPath string DomainName string BucketsPath string GrpcDialOption grpc.DialOption @@ -58,6 +59,11 @@ func NewWebDavServer(option *WebDavOption) (ws *WebDavServer, err error) { fs, _ := NewWebDavFileSystem(option) + // Fix no set filer.path , accessing "/" returns "//" + if option.FilerRootPath == "/" { + option.FilerRootPath = "" + } + ws = &WebDavServer{ option: option, grpcDialOption: security.LoadClientTLS(util.GetViper(), "grpc.filer"), @@ -195,7 +201,8 @@ func (fs *WebDavFileSystem) Mkdir(ctx context.Context, fullDirPath string, perm } func (fs *WebDavFileSystem) OpenFile(ctx context.Context, fullFilePath string, flag int, perm os.FileMode) (webdav.File, error) { - + // Add filer.path + fullFilePath = fs.option.FilerRootPath + fullFilePath glog.V(2).Infof("WebDavFileSystem.OpenFile %v %x", fullFilePath, flag) var err error @@ -367,7 +374,8 @@ func (fs *WebDavFileSystem) stat(ctx context.Context, fullFilePath string) (os.F } func (fs *WebDavFileSystem) Stat(ctx context.Context, name string) (os.FileInfo, error) { - + // Add filer.path + name = fs.option.FilerRootPath + name glog.V(2).Infof("WebDavFileSystem.Stat %v", name) return fs.stat(ctx, name) From b8a4ed455f338f70e6cddd8074164f36cfbab365 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 19 Dec 2022 09:21:20 -0800 Subject: [PATCH 03/25] build(deps): bump golang.org/x/tools from 0.3.0 to 0.4.0 (#4071) Bumps [golang.org/x/tools](https://github.com/golang/tools) from 0.3.0 to 0.4.0. - [Release notes](https://github.com/golang/tools/releases) - [Commits](https://github.com/golang/tools/compare/v0.3.0...v0.4.0) --- updated-dependencies: - dependency-name: golang.org/x/tools dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index d2fcb3c7f..ab9487edb 100644 --- a/go.mod +++ b/go.mod @@ -119,7 +119,7 @@ require ( golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783 // indirect golang.org/x/sys v0.3.0 golang.org/x/text v0.5.0 // indirect - golang.org/x/tools v0.3.0 + golang.org/x/tools v0.4.0 golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect google.golang.org/api v0.104.0 google.golang.org/appengine v1.6.7 // indirect diff --git a/go.sum b/go.sum index 3c462ad9a..cac846ff4 100644 --- a/go.sum +++ b/go.sum @@ -2313,8 +2313,8 @@ golang.org/x/tools v0.1.9/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= golang.org/x/tools v0.1.11/go.mod h1:SgwaegtQh8clINPpECJMqnxLv9I09HLqnW3RMqW0CA4= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.3.0 h1:SrNbZl6ECOS1qFzgTdQfWXZM9XBkiA6tkFrH9YSTPHM= -golang.org/x/tools v0.3.0/go.mod h1:/rWhSS2+zyEVwoJf8YAX6L2f0ntZ7Kn/mGgAWcipA5k= +golang.org/x/tools v0.4.0 h1:7mTAgkunk3fr4GAloyyCasadO6h9zSsQZbwvcaIciV4= +golang.org/x/tools v0.4.0/go.mod h1:UE5sM2OK9E/d67R0ANs2xJizIymRP5gJU295PvKXxjQ= golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= From 3b85dc5fbbdc37261e311923cfc4bbacf8841c7d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 19 Dec 2022 09:21:30 -0800 Subject: [PATCH 04/25] build(deps): bump github.com/ydb-platform/ydb-go-sdk/v3 from 3.40.1 to 3.41.0 (#4070) build(deps): bump github.com/ydb-platform/ydb-go-sdk/v3 Bumps [github.com/ydb-platform/ydb-go-sdk/v3](https://github.com/ydb-platform/ydb-go-sdk) from 3.40.1 to 3.41.0. - [Release notes](https://github.com/ydb-platform/ydb-go-sdk/releases) - [Changelog](https://github.com/ydb-platform/ydb-go-sdk/blob/master/CHANGELOG.md) - [Commits](https://github.com/ydb-platform/ydb-go-sdk/compare/v3.40.1...v3.41.0) --- updated-dependencies: - dependency-name: github.com/ydb-platform/ydb-go-sdk/v3 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index ab9487edb..b474ac7d9 100644 --- a/go.mod +++ b/go.mod @@ -151,7 +151,7 @@ require ( github.com/rabbitmq/amqp091-go v1.5.0 github.com/tikv/client-go/v2 v2.0.3 github.com/ydb-platform/ydb-go-sdk-auth-environ v0.1.2 - github.com/ydb-platform/ydb-go-sdk/v3 v3.40.1 + github.com/ydb-platform/ydb-go-sdk/v3 v3.41.0 golang.org/x/sync v0.1.0 google.golang.org/grpc/security/advancedtls v0.0.0-20220622233350-5cdb09fa29c1 ) @@ -218,7 +218,7 @@ require ( github.com/tinylib/msgp v1.1.6 // indirect github.com/twmb/murmur3 v1.1.3 // indirect github.com/yandex-cloud/go-genproto v0.0.0-20211115083454-9ca41db5ed9e // indirect - github.com/ydb-platform/ydb-go-genproto v0.0.0-20220922065549-66df47a830ba // indirect + github.com/ydb-platform/ydb-go-genproto v0.0.0-20221215182650-986f9d10542f // indirect github.com/ydb-platform/ydb-go-yc v0.8.3 // indirect github.com/ydb-platform/ydb-go-yc-metadata v0.5.2 // indirect go.etcd.io/bbolt v1.3.6 // indirect diff --git a/go.sum b/go.sum index cac846ff4..cc3046cb0 100644 --- a/go.sum +++ b/go.sum @@ -1682,13 +1682,13 @@ github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1: github.com/yandex-cloud/go-genproto v0.0.0-20211115083454-9ca41db5ed9e h1:9LPdmD1vqadsDQUva6t2O9MbnyvoOgo8nFNPaOIH5U8= github.com/yandex-cloud/go-genproto v0.0.0-20211115083454-9ca41db5ed9e/go.mod h1:HEUYX/p8966tMUHHT+TsS0hF/Ca/NYwqprC5WXSDMfE= github.com/ydb-platform/ydb-go-genproto v0.0.0-20220203104745-929cf9c248bc/go.mod h1:cc138nptTn9eKptCQl/grxP6pBKpo/bnXDiOxuVZtps= -github.com/ydb-platform/ydb-go-genproto v0.0.0-20220922065549-66df47a830ba h1:htUAISxEY5MfCLmubsf2EMgN+H62cQTNcTseK5F4cJ0= -github.com/ydb-platform/ydb-go-genproto v0.0.0-20220922065549-66df47a830ba/go.mod h1:Er+FePu1dNUieD+XTMDduGpQuCPssK5Q4BjF+IIXJ3I= +github.com/ydb-platform/ydb-go-genproto v0.0.0-20221215182650-986f9d10542f h1:BBczNIM1MJHT7XkIUA8pThXWxJvxoBjcWvne3xwe2RI= +github.com/ydb-platform/ydb-go-genproto v0.0.0-20221215182650-986f9d10542f/go.mod h1:Er+FePu1dNUieD+XTMDduGpQuCPssK5Q4BjF+IIXJ3I= github.com/ydb-platform/ydb-go-sdk-auth-environ v0.1.2 h1:EYSI1kulnHb0H0zt3yOw4cRj4ABMSMGwNe43D+fX7e4= github.com/ydb-platform/ydb-go-sdk-auth-environ v0.1.2/go.mod h1:Xfjce+VMU9yJVr1lj60yK2fFPWjB4jr/4cp3K7cjzi4= github.com/ydb-platform/ydb-go-sdk/v3 v3.25.3/go.mod h1:PFizF/vJsdAgEwjK3DVSBD52kdmRkWfSIS2q2pA+e88= -github.com/ydb-platform/ydb-go-sdk/v3 v3.40.1 h1:6a/yMcXT+ZpybYsGvEpFSsOdQlbqtArycUqY7U1Yd1w= -github.com/ydb-platform/ydb-go-sdk/v3 v3.40.1/go.mod h1:hJqWSE2NZ2o2c9geHtRJee+xwiHgEfQX9koBZPLTfHY= +github.com/ydb-platform/ydb-go-sdk/v3 v3.41.0 h1:2Kfj7I9EtQ8/o6gLB0mbShrZ/tC3/GFyjVE9z5CCLA0= +github.com/ydb-platform/ydb-go-sdk/v3 v3.41.0/go.mod h1:4bSfSb2PyBtmbFYsaVg96YxK1NjYBA3opn2o6IYvgZ8= github.com/ydb-platform/ydb-go-yc v0.8.3 h1:92UUUMsfvtMl6mho8eQ9lbkiPrF3a9CT+RrVRAKNRwo= github.com/ydb-platform/ydb-go-yc v0.8.3/go.mod h1:zUolAFGzJ5XG8uwiseTLr9Lapm7L7hdVdZgLSuv9FXE= github.com/ydb-platform/ydb-go-yc-metadata v0.5.2 h1:nMtixUijP0Z7iHJNT9fOL+dbmEzZxqU6Xk87ll7hqXg= From ae847c27807281d9443abc9c14c2629dd764785a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 19 Dec 2022 09:21:41 -0800 Subject: [PATCH 05/25] build(deps): bump actions/dependency-review-action from 3.0.1 to 3.0.2 (#4064) Bumps [actions/dependency-review-action](https://github.com/actions/dependency-review-action) from 3.0.1 to 3.0.2. - [Release notes](https://github.com/actions/dependency-review-action/releases) - [Commits](https://github.com/actions/dependency-review-action/compare/11310527b429536e263dc6cc47873e608189ba21...0ff3da6f81b812d4ec3cf37a04e2308c7a723730) --- updated-dependencies: - dependency-name: actions/dependency-review-action dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/depsreview.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/depsreview.yml b/.github/workflows/depsreview.yml index e6bd5fdd9..c8ab974d5 100644 --- a/.github/workflows/depsreview.yml +++ b/.github/workflows/depsreview.yml @@ -11,4 +11,4 @@ jobs: - name: 'Checkout Repository' uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 - name: 'Dependency Review' - uses: actions/dependency-review-action@11310527b429536e263dc6cc47873e608189ba21 + uses: actions/dependency-review-action@0ff3da6f81b812d4ec3cf37a04e2308c7a723730 From 6c5d44a5c49fd2475d028dabd4802522110ca124 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 19 Dec 2022 09:21:49 -0800 Subject: [PATCH 06/25] build(deps): bump actions/setup-go from 3.4.0 to 3.5.0 (#4065) Bumps [actions/setup-go](https://github.com/actions/setup-go) from 3.4.0 to 3.5.0. - [Release notes](https://github.com/actions/setup-go/releases) - [Commits](https://github.com/actions/setup-go/compare/d0a58c1c4d2b25278816e339b944508c875f3613...6edd4406fa81c3da01a34fa6f6343087c207a568) --- updated-dependencies: - dependency-name: actions/setup-go dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/e2e.yml | 2 +- .github/workflows/go.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 0650d253d..3dc61807a 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -24,7 +24,7 @@ jobs: timeout-minutes: 15 steps: - name: Set up Go 1.x - uses: actions/setup-go@d0a58c1c4d2b25278816e339b944508c875f3613 # v2 + uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # v2 with: go-version: ^1.13 id: go diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 9e48a67e8..a345d407b 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -21,7 +21,7 @@ jobs: steps: - name: Set up Go 1.x - uses: actions/setup-go@d0a58c1c4d2b25278816e339b944508c875f3613 # v2 + uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # v2 with: go-version: ^1.13 id: go From c7a28b20b5e339f627c030153b708a94a8ee7648 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 19 Dec 2022 09:22:12 -0800 Subject: [PATCH 07/25] build(deps): bump actions/checkout from 3.1.0 to 3.2.0 (#4066) Bumps [actions/checkout](https://github.com/actions/checkout) from 3.1.0 to 3.2.0. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8...755da8c3cf115ac066823e79a1e1788f8940201b) --- updated-dependencies: - dependency-name: actions/checkout dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/binaries_dev.yml | 4 ++-- .github/workflows/binaries_release0.yml | 2 +- .github/workflows/binaries_release1.yml | 2 +- .github/workflows/binaries_release2.yml | 2 +- .github/workflows/binaries_release3.yml | 2 +- .github/workflows/binaries_release4.yml | 2 +- .github/workflows/codeql.yml | 2 +- .github/workflows/container_dev.yml | 2 +- .github/workflows/container_latest.yml | 2 +- .github/workflows/container_release1.yml | 2 +- .github/workflows/container_release2.yml | 2 +- .github/workflows/container_release3.yml | 2 +- .github/workflows/container_release4.yml | 2 +- .github/workflows/container_release5.yml | 2 +- .github/workflows/depsreview.yml | 2 +- .github/workflows/e2e.yml | 2 +- .github/workflows/go.yml | 2 +- 17 files changed, 18 insertions(+), 18 deletions(-) diff --git a/.github/workflows/binaries_dev.yml b/.github/workflows/binaries_dev.yml index 743c3040c..a9f662fa2 100644 --- a/.github/workflows/binaries_dev.yml +++ b/.github/workflows/binaries_dev.yml @@ -38,7 +38,7 @@ jobs: steps: - name: Check out code into the Go module directory - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v2 + uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # v2 - name: Set BUILD_TIME env run: echo BUILD_TIME=$(date -u +%Y%m%d-%H%M) >> ${GITHUB_ENV} @@ -87,7 +87,7 @@ jobs: steps: - name: Check out code into the Go module directory - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v2 + uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # v2 - name: Set BUILD_TIME env run: echo BUILD_TIME=$(date -u +%Y%m%d-%H%M) >> ${GITHUB_ENV} diff --git a/.github/workflows/binaries_release0.yml b/.github/workflows/binaries_release0.yml index fa4c8b71b..aeac2b03d 100644 --- a/.github/workflows/binaries_release0.yml +++ b/.github/workflows/binaries_release0.yml @@ -28,7 +28,7 @@ jobs: # Steps represent a sequence of tasks that will be executed as part of the job steps: # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v2 + - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # v2 - name: Go Release Binaries Normal Volume Size uses: wangyoucao577/go-release-action@90da8ebfdc010a0e7d378419a76fd90230a05228 # v1.22 with: diff --git a/.github/workflows/binaries_release1.yml b/.github/workflows/binaries_release1.yml index 5387d6178..283f571d7 100644 --- a/.github/workflows/binaries_release1.yml +++ b/.github/workflows/binaries_release1.yml @@ -28,7 +28,7 @@ jobs: # Steps represent a sequence of tasks that will be executed as part of the job steps: # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v2 + - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # v2 - name: Go Release Binaries Normal Volume Size uses: wangyoucao577/go-release-action@90da8ebfdc010a0e7d378419a76fd90230a05228 # v1.22 with: diff --git a/.github/workflows/binaries_release2.yml b/.github/workflows/binaries_release2.yml index 71af542d3..6446bac59 100644 --- a/.github/workflows/binaries_release2.yml +++ b/.github/workflows/binaries_release2.yml @@ -28,7 +28,7 @@ jobs: # Steps represent a sequence of tasks that will be executed as part of the job steps: # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v2 + - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # v2 - name: Go Release Binaries Normal Volume Size uses: wangyoucao577/go-release-action@90da8ebfdc010a0e7d378419a76fd90230a05228 # v1.22 with: diff --git a/.github/workflows/binaries_release3.yml b/.github/workflows/binaries_release3.yml index b68a3fb6c..262ad7b17 100644 --- a/.github/workflows/binaries_release3.yml +++ b/.github/workflows/binaries_release3.yml @@ -28,7 +28,7 @@ jobs: # Steps represent a sequence of tasks that will be executed as part of the job steps: # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v2 + - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # v2 - name: Go Release Binaries Normal Volume Size uses: wangyoucao577/go-release-action@90da8ebfdc010a0e7d378419a76fd90230a05228 # v1.22 with: diff --git a/.github/workflows/binaries_release4.yml b/.github/workflows/binaries_release4.yml index 6d4728ea3..3f2df146e 100644 --- a/.github/workflows/binaries_release4.yml +++ b/.github/workflows/binaries_release4.yml @@ -28,7 +28,7 @@ jobs: # Steps represent a sequence of tasks that will be executed as part of the job steps: # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v2 + - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # v2 - name: Go Release Binaries Normal Volume Size uses: wangyoucao577/go-release-action@90da8ebfdc010a0e7d378419a76fd90230a05228 # v1.22 with: diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 1143d6e5e..292fda642 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -18,7 +18,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 + uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL diff --git a/.github/workflows/container_dev.yml b/.github/workflows/container_dev.yml index 3778d350b..6b01fb8ee 100644 --- a/.github/workflows/container_dev.yml +++ b/.github/workflows/container_dev.yml @@ -16,7 +16,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v2 + uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # v2 - name: Docker meta id: docker_meta diff --git a/.github/workflows/container_latest.yml b/.github/workflows/container_latest.yml index b3cec516e..1c365ea2d 100644 --- a/.github/workflows/container_latest.yml +++ b/.github/workflows/container_latest.yml @@ -17,7 +17,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v2 + uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # v2 - name: Docker meta id: docker_meta diff --git a/.github/workflows/container_release1.yml b/.github/workflows/container_release1.yml index c01d52853..e6e6dedaa 100644 --- a/.github/workflows/container_release1.yml +++ b/.github/workflows/container_release1.yml @@ -16,7 +16,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v2 + uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # v2 - name: Docker meta id: docker_meta diff --git a/.github/workflows/container_release2.yml b/.github/workflows/container_release2.yml index fcd16934d..a1264c4df 100644 --- a/.github/workflows/container_release2.yml +++ b/.github/workflows/container_release2.yml @@ -17,7 +17,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v2 + uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # v2 - name: Docker meta id: docker_meta diff --git a/.github/workflows/container_release3.yml b/.github/workflows/container_release3.yml index 138895281..4830f4f1b 100644 --- a/.github/workflows/container_release3.yml +++ b/.github/workflows/container_release3.yml @@ -17,7 +17,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v2 + uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # v2 - name: Docker meta id: docker_meta diff --git a/.github/workflows/container_release4.yml b/.github/workflows/container_release4.yml index b3dec5db3..c29f9ae5c 100644 --- a/.github/workflows/container_release4.yml +++ b/.github/workflows/container_release4.yml @@ -16,7 +16,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v2 + uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # v2 - name: Docker meta id: docker_meta diff --git a/.github/workflows/container_release5.yml b/.github/workflows/container_release5.yml index a91e9c4ed..d6aa985af 100644 --- a/.github/workflows/container_release5.yml +++ b/.github/workflows/container_release5.yml @@ -16,7 +16,7 @@ jobs: steps: - name: Checkout - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v2 + uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # v2 - name: Docker meta id: docker_meta diff --git a/.github/workflows/depsreview.yml b/.github/workflows/depsreview.yml index c8ab974d5..80a85ba4a 100644 --- a/.github/workflows/depsreview.yml +++ b/.github/workflows/depsreview.yml @@ -9,6 +9,6 @@ jobs: runs-on: ubuntu-latest steps: - name: 'Checkout Repository' - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 + uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b - name: 'Dependency Review' uses: actions/dependency-review-action@0ff3da6f81b812d4ec3cf37a04e2308c7a723730 diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 3dc61807a..3a2e9bff2 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -30,7 +30,7 @@ jobs: id: go - name: Check out code into the Go module directory - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v2 + uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # v2 - name: Install dependencies run: | diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index a345d407b..f46726743 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -27,7 +27,7 @@ jobs: id: go - name: Check out code into the Go module directory - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v2 + uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # v2 - name: Get dependencies run: | From 9fdf98b826e01d5aeed6712ac378ef6713ee6989 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 19 Dec 2022 09:22:23 -0800 Subject: [PATCH 08/25] build(deps): bump cloud.google.com/go/pubsub from 1.27.1 to 1.28.0 (#4068) Bumps [cloud.google.com/go/pubsub](https://github.com/googleapis/google-cloud-go) from 1.27.1 to 1.28.0. - [Release notes](https://github.com/googleapis/google-cloud-go/releases) - [Changelog](https://github.com/googleapis/google-cloud-go/blob/main/CHANGES.md) - [Commits](https://github.com/googleapis/google-cloud-go/compare/pubsub/v1.27.1...pubsub/v1.28.0) --- updated-dependencies: - dependency-name: cloud.google.com/go/pubsub dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index b474ac7d9..041adcc69 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.19 require ( cloud.google.com/go v0.105.0 // indirect - cloud.google.com/go/pubsub v1.27.1 + cloud.google.com/go/pubsub v1.28.0 cloud.google.com/go/storage v1.28.1 github.com/Azure/azure-pipeline-go v0.2.3 github.com/Azure/azure-storage-blob-go v0.15.0 diff --git a/go.sum b/go.sum index cc3046cb0..1585913c3 100644 --- a/go.sum +++ b/go.sum @@ -73,8 +73,8 @@ cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+ cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= cloud.google.com/go/pubsub v1.24.0/go.mod h1:rWv09Te1SsRpRGPiWOMDKraMQTJyJps4MkUCoMGUgqw= -cloud.google.com/go/pubsub v1.27.1 h1:q+J/Nfr6Qx4RQeu3rJcnN48SNC0qzlYzSeqkPq93VHs= -cloud.google.com/go/pubsub v1.27.1/go.mod h1:hQN39ymbV9geqBnfQq6Xf63yNhUAhv9CZhzp5O6qsW0= +cloud.google.com/go/pubsub v1.28.0 h1:XzabfdPx/+eNrsVVGLFgeUnQQKPGkMb8klRCeYK52is= +cloud.google.com/go/pubsub v1.28.0/go.mod h1:vuXFpwaVoIPQMGXqRyUQigu/AX1S3IWugR9xznmcXX8= cloud.google.com/go/secretmanager v1.5.0/go.mod h1:5C9kM+RwSpkURNovKySkNvGQLUaOgyoR5W0RUx2SyHQ= cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= From 3886c31ed8fb04bdf7e9a6818a2fe5bf62cd31cf Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 19 Dec 2022 09:22:36 -0800 Subject: [PATCH 09/25] build(deps): bump github.com/aws/aws-sdk-go from 1.44.146 to 1.44.162 (#4069) Bumps [github.com/aws/aws-sdk-go](https://github.com/aws/aws-sdk-go) from 1.44.146 to 1.44.162. - [Release notes](https://github.com/aws/aws-sdk-go/releases) - [Commits](https://github.com/aws/aws-sdk-go/compare/v1.44.146...v1.44.162) --- updated-dependencies: - dependency-name: github.com/aws/aws-sdk-go dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 041adcc69..cbd7cde78 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/Azure/azure-pipeline-go v0.2.3 github.com/Azure/azure-storage-blob-go v0.15.0 github.com/Shopify/sarama v1.37.2 - github.com/aws/aws-sdk-go v1.44.146 + github.com/aws/aws-sdk-go v1.44.162 github.com/beorn7/perks v1.0.1 // indirect github.com/bwmarrin/snowflake v0.3.0 github.com/cespare/xxhash/v2 v2.1.2 // indirect diff --git a/go.sum b/go.sum index 1585913c3..43f76c4f7 100644 --- a/go.sum +++ b/go.sum @@ -239,8 +239,8 @@ github.com/aws/aws-sdk-go v1.43.11/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4 github.com/aws/aws-sdk-go v1.43.31/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= github.com/aws/aws-sdk-go v1.44.45/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= github.com/aws/aws-sdk-go v1.44.68/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= -github.com/aws/aws-sdk-go v1.44.146 h1:7YdGgPxDPRJu/yYffzZp/H7yHzQ6AqmuNFZPYraaN8I= -github.com/aws/aws-sdk-go v1.44.146/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= +github.com/aws/aws-sdk-go v1.44.162 h1:hKAd+X+/BLxVMzH+4zKxbQcQQGrk2UhFX0OTu1Mhon8= +github.com/aws/aws-sdk-go v1.44.162/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= github.com/aws/aws-sdk-go-v2 v1.16.8 h1:gOe9UPR98XSf7oEJCcojYg+N2/jCRm4DdeIsP85pIyQ= github.com/aws/aws-sdk-go-v2 v1.16.8/go.mod h1:6CpKuLXg2w7If3ABZCl/qZ6rEgwtjZTn4eAf4RcEyuw= From a1ffa03d75c7e87950f9369ad16c21da81d56bc8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 19 Dec 2022 10:21:20 -0800 Subject: [PATCH 10/25] build(deps): bump google.golang.org/api from 0.104.0 to 0.105.0 (#4067) Bumps [google.golang.org/api](https://github.com/googleapis/google-api-go-client) from 0.104.0 to 0.105.0. - [Release notes](https://github.com/googleapis/google-api-go-client/releases) - [Changelog](https://github.com/googleapis/google-api-go-client/blob/main/CHANGES.md) - [Commits](https://github.com/googleapis/google-api-go-client/compare/v0.104.0...v0.105.0) --- updated-dependencies: - dependency-name: google.golang.org/api dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index cbd7cde78..cfd64c75a 100644 --- a/go.mod +++ b/go.mod @@ -121,7 +121,7 @@ require ( golang.org/x/text v0.5.0 // indirect golang.org/x/tools v0.4.0 golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect - google.golang.org/api v0.104.0 + google.golang.org/api v0.105.0 google.golang.org/appengine v1.6.7 // indirect google.golang.org/genproto v0.0.0-20221206210731-b1a01be3a5f6 // indirect google.golang.org/grpc v1.51.0 diff --git a/go.sum b/go.sum index 43f76c4f7..64551e04b 100644 --- a/go.sum +++ b/go.sum @@ -2374,8 +2374,8 @@ google.golang.org/api v0.85.0/go.mod h1:AqZf8Ep9uZ2pyTvgL+x0D3Zt0eoT9b5E8fmzfu6F google.golang.org/api v0.86.0/go.mod h1:+Sem1dnrKlrXMR/X0bPnMWyluQe4RsNoYfmNLhOIkzw= google.golang.org/api v0.90.0/go.mod h1:+Sem1dnrKlrXMR/X0bPnMWyluQe4RsNoYfmNLhOIkzw= google.golang.org/api v0.91.0/go.mod h1:+Sem1dnrKlrXMR/X0bPnMWyluQe4RsNoYfmNLhOIkzw= -google.golang.org/api v0.104.0 h1:KBfmLRqdZEbwQleFlSLnzpQJwhjpmNOk4cKQIBDZ9mg= -google.golang.org/api v0.104.0/go.mod h1:JCspTXJbBxa5ySXw4UgUqVer7DfVxbvc/CTUFqAED5U= +google.golang.org/api v0.105.0 h1:t6P9Jj+6XTn4U9I2wycQai6Q/Kz7iOT+QzjJ3G2V4x8= +google.golang.org/api v0.105.0/go.mod h1:qh7eD5FJks5+BcE+cjBIm6Gz8vioK7EHvnlniqXBnqI= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= From 3fc3e7083c442aefd009c95bdad7379afa660392 Mon Sep 17 00:00:00 2001 From: chrislu Date: Mon, 19 Dec 2022 11:30:50 -0800 Subject: [PATCH 11/25] metadata skip reading un-available logs this means the volumes for metadata histories have been lost. --- weed/filer/filer_notify.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/weed/filer/filer_notify.go b/weed/filer/filer_notify.go index 77b659288..f8a1dd603 100644 --- a/weed/filer/filer_notify.go +++ b/weed/filer/filer_notify.go @@ -5,6 +5,7 @@ import ( "fmt" "io" "math" + "regexp" "strings" "time" @@ -108,6 +109,10 @@ func (f *Filer) logFlushFunc(startTime, stopTime time.Time, buf []byte) { } } +var ( + VolumeNotFoundPattern = regexp.MustCompile(`volume \d+? not found`) +) + func (f *Filer) ReadPersistedLogBuffer(startTime time.Time, stopTsNs int64, eachLogEntryFn func(logEntry *filer_pb.LogEntry) error) (lastTsNs int64, isDone bool, err error) { startTime = startTime.UTC() @@ -159,6 +164,10 @@ func (f *Filer) ReadPersistedLogBuffer(startTime time.Time, stopTsNs int64, each if err == io.EOF { continue } + if VolumeNotFoundPattern.MatchString(err.Error()) { + glog.Warningf("skipping reading %s: %v", hourMinuteEntry.FullPath, err) + continue + } return lastTsNs, isDone, fmt.Errorf("reading %s: %v", hourMinuteEntry.FullPath, err) } chunkedFileReader.Close() From 6c7fe40305fdebe0902e44ae2a841ca4382b6a9b Mon Sep 17 00:00:00 2001 From: chrislu Date: Mon, 19 Dec 2022 11:31:58 -0800 Subject: [PATCH 12/25] filer sink retries reading file chunks, skipping missing chunks if the file chunk is not available during replication time, the file is skipped --- weed/replication/sink/filersink/fetch_write.go | 16 ++++++++++------ weed/replication/sink/filersink/filer_sink.go | 8 +++++--- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/weed/replication/sink/filersink/fetch_write.go b/weed/replication/sink/filersink/fetch_write.go index cd961f147..7d526513e 100644 --- a/weed/replication/sink/filersink/fetch_write.go +++ b/weed/replication/sink/filersink/fetch_write.go @@ -27,12 +27,16 @@ func (fs *FilerSink) replicateChunks(sourceChunks []*filer_pb.FileChunk, path st index, source := chunkIndex, sourceChunk fs.executor.Execute(func() { defer wg.Done() - replicatedChunk, e := fs.replicateOneChunk(source, path) - if e != nil { - err = e - return - } - replicatedChunks[index] = replicatedChunk + util.Retry("replicate chunks", func() error { + replicatedChunk, e := fs.replicateOneChunk(source, path) + if e != nil { + err = e + return e + } + replicatedChunks[index] = replicatedChunk + err = nil + return nil + }) }) } wg.Wait() diff --git a/weed/replication/sink/filersink/filer_sink.go b/weed/replication/sink/filersink/filer_sink.go index de5ff55cc..35b6ffa73 100644 --- a/weed/replication/sink/filersink/filer_sink.go +++ b/weed/replication/sink/filersink/filer_sink.go @@ -112,7 +112,7 @@ func (fs *FilerSink) CreateEntry(key string, entry *filer_pb.Entry, signatures [ Directory: dir, Name: name, } - glog.V(1).Infof("lookup: %v", lookupRequest) + // glog.V(1).Infof("lookup: %v", lookupRequest) if resp, err := filer_pb.LookupEntry(client, lookupRequest); err == nil { if filer.ETag(resp.Entry) == filer.ETag(entry) { glog.V(3).Infof("already replicated %s", key) @@ -125,9 +125,10 @@ func (fs *FilerSink) CreateEntry(key string, entry *filer_pb.Entry, signatures [ if err != nil { // only warning here since the source chunk may have been deleted already glog.Warningf("replicate entry chunks %s: %v", key, err) + return nil } - glog.V(4).Infof("replicated %s %+v ===> %+v", key, entry.GetChunks(), replicatedChunks) + // glog.V(4).Infof("replicated %s %+v ===> %+v", key, entry.GetChunks(), replicatedChunks) request := &filer_pb.CreateEntryRequest{ Directory: dir, @@ -205,7 +206,8 @@ func (fs *FilerSink) UpdateEntry(key string, oldEntry *filer_pb.Entry, newParent // replicate the chunks that are new in the source replicatedChunks, err := fs.replicateChunks(newChunks, key) if err != nil { - return true, fmt.Errorf("replicate %s chunks error: %v", key, err) + glog.Warningf("replicate entry chunks %s: %v", key, err) + return true, nil } existingEntry.Chunks = append(existingEntry.GetChunks(), replicatedChunks...) existingEntry.Attributes = newEntry.Attributes From 28d479e5c059bdb041aeaff1075a738990740bf2 Mon Sep 17 00:00:00 2001 From: chrislu Date: Mon, 19 Dec 2022 15:07:22 -0800 Subject: [PATCH 13/25] mount: adjust locking for upload pipeline --- weed/mount/page_writer/upload_pipeline.go | 64 +++++++++---------- .../mount/page_writer/upload_pipeline_lock.go | 6 -- 2 files changed, 31 insertions(+), 39 deletions(-) diff --git a/weed/mount/page_writer/upload_pipeline.go b/weed/mount/page_writer/upload_pipeline.go index 586e05b3f..f54a87a02 100644 --- a/weed/mount/page_writer/upload_pipeline.go +++ b/weed/mount/page_writer/upload_pipeline.go @@ -6,26 +6,24 @@ import ( "github.com/seaweedfs/seaweedfs/weed/util" "sync" "sync/atomic" - "time" ) type LogicChunkIndex int type UploadPipeline struct { - uploaderCount int32 - uploaderCountCond *sync.Cond - filepath util.FullPath - ChunkSize int64 - writableChunks map[LogicChunkIndex]PageChunk - writableChunksLock sync.Mutex - sealedChunks map[LogicChunkIndex]*SealedChunk - sealedChunksLock sync.Mutex - uploaders *util.LimitedConcurrentExecutor - saveToStorageFn SaveToStorageFunc - activeReadChunks map[LogicChunkIndex]int - activeReadChunksLock sync.Mutex - writableChunkLimit int - swapFile *SwapFile + uploaderCount int32 + uploaderCountCond *sync.Cond + filepath util.FullPath + ChunkSize int64 + uploaders *util.LimitedConcurrentExecutor + saveToStorageFn SaveToStorageFunc + writableChunkLimit int + swapFile *SwapFile + chunksLock sync.Mutex + writableChunks map[LogicChunkIndex]PageChunk + sealedChunks map[LogicChunkIndex]*SealedChunk + activeReadChunks map[LogicChunkIndex]int + readerCountCond *sync.Cond } type SealedChunk struct { @@ -42,7 +40,7 @@ func (sc *SealedChunk) FreeReference(messageOnFree string) { } func NewUploadPipeline(writers *util.LimitedConcurrentExecutor, chunkSize int64, saveToStorageFn SaveToStorageFunc, bufferChunkLimit int, swapFileDir string) *UploadPipeline { - return &UploadPipeline{ + t := &UploadPipeline{ ChunkSize: chunkSize, writableChunks: make(map[LogicChunkIndex]PageChunk), sealedChunks: make(map[LogicChunkIndex]*SealedChunk), @@ -53,11 +51,13 @@ func NewUploadPipeline(writers *util.LimitedConcurrentExecutor, chunkSize int64, writableChunkLimit: bufferChunkLimit, swapFile: NewSwapFile(swapFileDir, chunkSize), } + t.readerCountCond = sync.NewCond(&t.chunksLock) + return t } func (up *UploadPipeline) SaveDataAt(p []byte, off int64, isSequential bool) (n int) { - up.writableChunksLock.Lock() - defer up.writableChunksLock.Unlock() + up.chunksLock.Lock() + defer up.chunksLock.Unlock() logicChunkIndex := LogicChunkIndex(off / up.ChunkSize) @@ -95,13 +95,17 @@ func (up *UploadPipeline) SaveDataAt(p []byte, off int64, isSequential bool) (n func (up *UploadPipeline) MaybeReadDataAt(p []byte, off int64) (maxStop int64) { logicChunkIndex := LogicChunkIndex(off / up.ChunkSize) + up.chunksLock.Lock() + defer func() { + up.readerCountCond.Signal() + up.chunksLock.Unlock() + }() + // read from sealed chunks first - up.sealedChunksLock.Lock() sealedChunk, found := up.sealedChunks[logicChunkIndex] if found { sealedChunk.referenceCounter++ } - up.sealedChunksLock.Unlock() if found { maxStop = sealedChunk.chunk.ReadDataAt(p, off) glog.V(4).Infof("%s read sealed memchunk [%d,%d)", up.filepath, off, maxStop) @@ -109,8 +113,6 @@ func (up *UploadPipeline) MaybeReadDataAt(p []byte, off int64) (maxStop int64) { } // read from writable chunks last - up.writableChunksLock.Lock() - defer up.writableChunksLock.Unlock() writableChunk, found := up.writableChunks[logicChunkIndex] if !found { return @@ -123,8 +125,8 @@ func (up *UploadPipeline) MaybeReadDataAt(p []byte, off int64) (maxStop int64) { } func (up *UploadPipeline) FlushAll() { - up.writableChunksLock.Lock() - defer up.writableChunksLock.Unlock() + up.chunksLock.Lock() + defer up.chunksLock.Unlock() for logicChunkIndex, memChunk := range up.writableChunks { up.moveToSealed(memChunk, logicChunkIndex) @@ -143,8 +145,6 @@ func (up *UploadPipeline) moveToSealed(memChunk PageChunk, logicChunkIndex Logic atomic.AddInt32(&up.uploaderCount, 1) glog.V(4).Infof("%s uploaderCount %d ++> %d", up.filepath, up.uploaderCount-1, up.uploaderCount) - up.sealedChunksLock.Lock() - if oldMemChunk, found := up.sealedChunks[logicChunkIndex]; found { oldMemChunk.FreeReference(fmt.Sprintf("%s replace chunk %d", up.filepath, logicChunkIndex)) } @@ -155,8 +155,6 @@ func (up *UploadPipeline) moveToSealed(memChunk PageChunk, logicChunkIndex Logic up.sealedChunks[logicChunkIndex] = sealedChunk delete(up.writableChunks, logicChunkIndex) - up.sealedChunksLock.Unlock() - up.uploaders.Execute(func() { // first add to the file chunks sealedChunk.chunk.SaveContent(up.saveToStorageFn) @@ -172,13 +170,13 @@ func (up *UploadPipeline) moveToSealed(memChunk PageChunk, logicChunkIndex Logic up.uploaderCountCond.L.Unlock() // wait for readers + up.chunksLock.Lock() + defer up.chunksLock.Unlock() for up.IsLocked(logicChunkIndex) { - time.Sleep(59 * time.Millisecond) + up.readerCountCond.Wait() } // then remove from sealed chunks - up.sealedChunksLock.Lock() - defer up.sealedChunksLock.Unlock() delete(up.sealedChunks, logicChunkIndex) sealedChunk.FreeReference(fmt.Sprintf("%s finished uploading chunk %d", up.filepath, logicChunkIndex)) @@ -188,8 +186,8 @@ func (up *UploadPipeline) moveToSealed(memChunk PageChunk, logicChunkIndex Logic func (up *UploadPipeline) Shutdown() { up.swapFile.FreeResource() - up.sealedChunksLock.Lock() - defer up.sealedChunksLock.Unlock() + up.chunksLock.Lock() + defer up.chunksLock.Unlock() for logicChunkIndex, sealedChunk := range up.sealedChunks { sealedChunk.FreeReference(fmt.Sprintf("%s uploadpipeline shutdown chunk %d", up.filepath, logicChunkIndex)) } diff --git a/weed/mount/page_writer/upload_pipeline_lock.go b/weed/mount/page_writer/upload_pipeline_lock.go index 47a40ba37..43b04e01d 100644 --- a/weed/mount/page_writer/upload_pipeline_lock.go +++ b/weed/mount/page_writer/upload_pipeline_lock.go @@ -10,8 +10,6 @@ func (up *UploadPipeline) LockForRead(startOffset, stopOffset int64) { if stopOffset%up.ChunkSize > 0 { stopLogicChunkIndex += 1 } - up.activeReadChunksLock.Lock() - defer up.activeReadChunksLock.Unlock() for i := startLogicChunkIndex; i < stopLogicChunkIndex; i++ { if count, found := up.activeReadChunks[i]; found { up.activeReadChunks[i] = count + 1 @@ -27,8 +25,6 @@ func (up *UploadPipeline) UnlockForRead(startOffset, stopOffset int64) { if stopOffset%up.ChunkSize > 0 { stopLogicChunkIndex += 1 } - up.activeReadChunksLock.Lock() - defer up.activeReadChunksLock.Unlock() for i := startLogicChunkIndex; i < stopLogicChunkIndex; i++ { if count, found := up.activeReadChunks[i]; found { if count == 1 { @@ -41,8 +37,6 @@ func (up *UploadPipeline) UnlockForRead(startOffset, stopOffset int64) { } func (up *UploadPipeline) IsLocked(logicChunkIndex LogicChunkIndex) bool { - up.activeReadChunksLock.Lock() - defer up.activeReadChunksLock.Unlock() if count, found := up.activeReadChunks[logicChunkIndex]; found { return count > 0 } From f7beba8515ddfc138f7d068207baa0765c03ec0c Mon Sep 17 00:00:00 2001 From: chrislu Date: Tue, 20 Dec 2022 12:52:30 -0800 Subject: [PATCH 14/25] unlock before submitting the uploading jobs --- weed/mount/page_writer/upload_pipeline.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/weed/mount/page_writer/upload_pipeline.go b/weed/mount/page_writer/upload_pipeline.go index f54a87a02..bce062a6e 100644 --- a/weed/mount/page_writer/upload_pipeline.go +++ b/weed/mount/page_writer/upload_pipeline.go @@ -73,8 +73,9 @@ func (up *UploadPipeline) SaveDataAt(p []byte, off int64, isSequential bool) (n fullness = chunkFullness } } - up.moveToSealed(up.writableChunks[fullestChunkIndex], fullestChunkIndex) + fullWritableChunk := up.writableChunks[fullestChunkIndex] delete(up.writableChunks, fullestChunkIndex) + up.moveToSealed(fullWritableChunk, fullestChunkIndex) // fmt.Printf("flush chunk %d with %d bytes written\n", logicChunkIndex, fullness) } if isSequential && @@ -155,6 +156,8 @@ func (up *UploadPipeline) moveToSealed(memChunk PageChunk, logicChunkIndex Logic up.sealedChunks[logicChunkIndex] = sealedChunk delete(up.writableChunks, logicChunkIndex) + // unlock before submitting the uploading jobs + up.chunksLock.Unlock() up.uploaders.Execute(func() { // first add to the file chunks sealedChunk.chunk.SaveContent(up.saveToStorageFn) @@ -181,6 +184,7 @@ func (up *UploadPipeline) moveToSealed(memChunk PageChunk, logicChunkIndex Logic sealedChunk.FreeReference(fmt.Sprintf("%s finished uploading chunk %d", up.filepath, logicChunkIndex)) }) + up.chunksLock.Lock() } func (up *UploadPipeline) Shutdown() { From 3b90eb1911225523821bba85b5dbbc04d29cb9f1 Mon Sep 17 00:00:00 2001 From: Naden Date: Wed, 21 Dec 2022 12:24:12 +1100 Subject: [PATCH 15/25] Fixes: 4076: Host path prefix should be configurable (#4077) Co-authored-by: Naden --- k8s/helm_charts2/templates/filer-statefulset.yaml | 4 ++-- k8s/helm_charts2/templates/master-statefulset.yaml | 4 ++-- k8s/helm_charts2/templates/s3-deployment.yaml | 2 +- k8s/helm_charts2/templates/volume-statefulset.yaml | 6 +++--- k8s/helm_charts2/values.yaml | 10 +++++++++- 5 files changed, 17 insertions(+), 9 deletions(-) diff --git a/k8s/helm_charts2/templates/filer-statefulset.yaml b/k8s/helm_charts2/templates/filer-statefulset.yaml index 689c1092c..978cdeec8 100644 --- a/k8s/helm_charts2/templates/filer-statefulset.yaml +++ b/k8s/helm_charts2/templates/filer-statefulset.yaml @@ -225,13 +225,13 @@ spec: {{- if eq .Values.filer.logs.type "hostPath" }} - name: seaweedfs-filer-log-volume hostPath: - path: /storage/logs/seaweedfs/filer + path: {{ .Values.filer.logs.hostPathPrefix }}/logs/seaweedfs/filer type: DirectoryOrCreate {{- end }} {{- if eq .Values.filer.data.type "hostPath" }} - name: data-filer hostPath: - path: /storage/filer_store + path: {{ .Values.filer.data.hostPathPrefix }}/filer_store type: DirectoryOrCreate {{- end }} - name: db-schema-config-volume diff --git a/k8s/helm_charts2/templates/master-statefulset.yaml b/k8s/helm_charts2/templates/master-statefulset.yaml index 0b58843fd..c31962036 100644 --- a/k8s/helm_charts2/templates/master-statefulset.yaml +++ b/k8s/helm_charts2/templates/master-statefulset.yaml @@ -196,13 +196,13 @@ spec: {{- if eq .Values.master.logs.type "hostPath" }} - name: seaweedfs-master-log-volume hostPath: - path: /storage/logs/seaweedfs/master + path: {{ .Values.master.logs.hostPathPrefix }}/logs/seaweedfs/master type: DirectoryOrCreate {{- end }} {{- if eq .Values.master.data.type "hostPath" }} - name: data-{{ .Release.Namespace }} hostPath: - path: /ssd/seaweed-master/ + path: {{ .Values.master.data.hostPathPrefix }}/seaweed-master/ type: DirectoryOrCreate {{- end }} {{- if .Values.global.enableSecurity }} diff --git a/k8s/helm_charts2/templates/s3-deployment.yaml b/k8s/helm_charts2/templates/s3-deployment.yaml index b9efef827..02b0e0046 100644 --- a/k8s/helm_charts2/templates/s3-deployment.yaml +++ b/k8s/helm_charts2/templates/s3-deployment.yaml @@ -167,7 +167,7 @@ spec: {{- if eq .Values.s3.logs.type "hostPath" }} - name: logs hostPath: - path: /storage/logs/seaweedfs/s3 + path: {{ .Values.s3.logs.hostPathPrefix }}/logs/seaweedfs/s3 type: DirectoryOrCreate {{- end }} {{- if .Values.global.enableSecurity }} diff --git a/k8s/helm_charts2/templates/volume-statefulset.yaml b/k8s/helm_charts2/templates/volume-statefulset.yaml index 0201c9cc4..55431f477 100644 --- a/k8s/helm_charts2/templates/volume-statefulset.yaml +++ b/k8s/helm_charts2/templates/volume-statefulset.yaml @@ -206,19 +206,19 @@ spec: {{- if eq .Values.volume.data.type "hostPath" }} - name: data hostPath: - path: /storage/object_store/ + path: {{ .Values.volume.data.hostPathPrefix }}/object_store/ type: DirectoryOrCreate {{- end }} {{- if and (eq .Values.volume.idx.type "hostPath") .Values.volume.dir_idx }} - name: idx hostPath: - path: /ssd/seaweedfs-volume-idx/ + path: {{ .Values.volume.idx.hostPathPrefix }}/seaweedfs-volume-idx/ type: DirectoryOrCreate {{- end }} {{- if eq .Values.volume.logs.type "hostPath" }} - name: logs hostPath: - path: /storage/logs/seaweedfs/volume + path: {{ .Values.volume.logs.hostPathPrefix }}/logs/seaweedfs/volume type: DirectoryOrCreate {{- end }} {{- if .Values.global.enableSecurity }} diff --git a/k8s/helm_charts2/values.yaml b/k8s/helm_charts2/values.yaml index 02978927b..4feacc7bf 100644 --- a/k8s/helm_charts2/values.yaml +++ b/k8s/helm_charts2/values.yaml @@ -68,11 +68,13 @@ master: type: "hostPath" size: "" storageClass: "" + hostPathPrefix: /ssd logs: type: "hostPath" size: "" storageClass: "" + hostPathPrefix: /storage initContainers: "" @@ -176,15 +178,19 @@ volume: type: "hostPath" size: "" storageClass: "" + hostPathPrefix: /storage + idx: type: "hostPath" size: "" storageClass: "" + hostPathPrefix: /ssd logs: type: "hostPath" size: "" storageClass: "" + hostPathPrefix: /storage # limit background compaction or copying speed in mega bytes per second compactionMBps: "50" @@ -308,11 +314,13 @@ filer: type: "hostPath" size: "" storageClass: "" + hostPathPrefix: /storage logs: type: "hostPath" size: "" storageClass: "" + hostPathPrefix: /storage initContainers: "" @@ -463,7 +471,7 @@ s3: type: "hostPath" size: "" storageClass: "" - + hostPathPrefix: /storage certificates: commonName: "SeaweedFS CA" From 27d187eed9285af234e0eafbf97450080f91cb2d Mon Sep 17 00:00:00 2001 From: Naden Date: Wed, 21 Dec 2022 12:24:54 +1100 Subject: [PATCH 16/25] Fixes 4074: S3 Gateway fails to start when installed via Helm (#4075) Fixes #4074: S3 Gateway fails to start when installed via Helm Co-authored-by: Naden --- k8s/helm_charts2/templates/s3-deployment.yaml | 1 + k8s/helm_charts2/values.yaml | 1 + 2 files changed, 2 insertions(+) diff --git a/k8s/helm_charts2/templates/s3-deployment.yaml b/k8s/helm_charts2/templates/s3-deployment.yaml index 02b0e0046..c9e12e9cb 100644 --- a/k8s/helm_charts2/templates/s3-deployment.yaml +++ b/k8s/helm_charts2/templates/s3-deployment.yaml @@ -79,6 +79,7 @@ spec: -v={{ .Values.global.loggingLevel }} \ {{- end }} s3 \ + -ip.bind={{ .Values.s3.bindAddress }} \ -port={{ .Values.s3.port }} \ {{- if .Values.s3.metricsPort }} -metricsPort {{ .Values.s3.metricsPort }} \ diff --git a/k8s/helm_charts2/values.yaml b/k8s/helm_charts2/values.yaml index 4feacc7bf..21d031eb5 100644 --- a/k8s/helm_charts2/values.yaml +++ b/k8s/helm_charts2/values.yaml @@ -426,6 +426,7 @@ s3: imageTag: null restartPolicy: null replicas: 1 + bindAddress: 0.0.0.0 port: 8333 metricsPort: 9327 loggingOverrideLevel: null From 6ede19e8257205fa72c6da46bf74382ada44b6be Mon Sep 17 00:00:00 2001 From: chrislu Date: Tue, 20 Dec 2022 19:47:21 -0800 Subject: [PATCH 17/25] add a simple file replication progress bar --- go.mod | 8 ++++++- go.sum | 15 ++++++++++++ .../replication/sink/filersink/fetch_write.go | 23 +++++++++++++++++-- 3 files changed, 43 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index cfd64c75a..01a1f0035 100644 --- a/go.mod +++ b/go.mod @@ -194,10 +194,12 @@ require ( github.com/jcmturner/rpc/v2 v2.0.3 // indirect github.com/jonboulle/clockwork v0.2.2 // indirect github.com/josharian/intern v1.0.0 // indirect + github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213 // indirect github.com/klauspost/cpuid/v2 v2.1.1 // indirect github.com/mattn/go-colorable v0.1.13 // indirect - github.com/mattn/go-runewidth v0.0.7 // indirect + github.com/mattn/go-runewidth v0.0.14 // indirect github.com/mattn/go-sqlite3 v2.0.1+incompatible // indirect + github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect github.com/montanaflynn/stats v0.6.6 // indirect github.com/nats-io/nats.go v1.16.0 // indirect github.com/nats-io/nkeys v0.3.0 // indirect @@ -210,7 +212,10 @@ require ( github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00 // indirect github.com/pingcap/kvproto v0.0.0-20221129023506-621ec37aac7a // indirect github.com/pingcap/log v1.1.1-0.20221015072633-39906604fb81 // indirect + github.com/rivo/uniseg v0.4.3 // indirect github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd // indirect + github.com/schollz/progressbar v1.0.0 // indirect + github.com/schollz/progressbar/v3 v3.12.2 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/stathat/consistent v1.0.0 // indirect github.com/subosito/gotenv v1.4.1 // indirect @@ -228,6 +233,7 @@ require ( go.uber.org/multierr v1.8.0 // indirect go.uber.org/zap v1.22.0 // indirect golang.org/x/mod v0.7.0 // indirect + golang.org/x/term v0.3.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect diff --git a/go.sum b/go.sum index 64551e04b..ddd76311f 100644 --- a/go.sum +++ b/go.sum @@ -1092,6 +1092,8 @@ github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/X github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= +github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213 h1:qGQQKEcAR99REcMpsXCp3lJ03zYT1PkRd3kQGPn9GVg= +github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213/go.mod h1:vNUNkEQ1e29fT/6vq2aBdFsgNPmy8qMdSay1npru+Sw= github.com/karlseguin/ccache/v2 v2.0.8 h1:lT38cE//uyf6KcFok0rlgXtGFBWxkI6h/qg4tbFyDnA= github.com/karlseguin/ccache/v2 v2.0.8/go.mod h1:2BDThcfQMf/c0jnZowt16eW405XIqZPavt+HoYEtcxQ= github.com/karlseguin/expect v1.0.2-0.20190806010014-778a5f0c6003 h1:vJ0Snvo+SLMY72r5J4sEfkuE7AFbixEP2qRbEcum/wA= @@ -1196,6 +1198,8 @@ github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzp github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.7 h1:Ei8KR0497xHyKJPAv59M1dkC+rOZCMBJ+t3fZ+twI54= github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= +github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU= +github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mattn/go-shellwords v1.0.3/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vqg+NOMyg4B2o= github.com/mattn/go-shellwords v1.0.6/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vqg+NOMyg4B2o= github.com/mattn/go-shellwords v1.0.12/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lLtQsUlTZDWQ8Y= @@ -1218,6 +1222,8 @@ github.com/minio/highwayhash v1.0.2/go.mod h1:BQskDq+xkJ12lmlUUi7U0M5Swg3EWR+dLT github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible/go.mod h1:8AuVvqP/mXw1px98n46wfvcGfQ4ci2FwoAjKYxuo3Z4= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/cli v1.1.0/go.mod h1:xcISNoH86gajksDmfB23e/pu+B+GeFRMYmoHXxx3xhI= +github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ= +github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw= github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= @@ -1491,6 +1497,9 @@ github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5X github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 h1:OdAsTTz6OkFY5QxjkYwrChwuRruF69c169dPK26NUlk= github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= +github.com/rivo/uniseg v0.4.3 h1:utMvzDsuh3suAEnhH0RdHmoPbU648o6CvXxTx4SBMOw= +github.com/rivo/uniseg v0.4.3/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= @@ -1513,6 +1522,10 @@ github.com/safchain/ethtool v0.0.0-20210803160452-9aa261dae9b1/go.mod h1:Z0q5wiB github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/scaleway/scaleway-sdk-go v1.0.0-beta.9/go.mod h1:fCa7OJZ/9DRTnOKmxvT6pn+LPWUptQAmHF/SBJUGEcg= +github.com/schollz/progressbar v1.0.0 h1:gbyFReLHDkZo8mxy/dLWMr+Mpb1MokGJ1FqCiqacjZM= +github.com/schollz/progressbar v1.0.0/go.mod h1:/l9I7PC3L3erOuz54ghIRKUEFcosiWfLvJv+Eq26UMs= +github.com/schollz/progressbar/v3 v3.12.2 h1:yLqqqpQNMxGxHY8uEshRihaHWwa0rf0yb7/Zrpgq2C0= +github.com/schollz/progressbar/v3 v3.12.2/go.mod h1:HFJYIYQQJX32UJdyoigUl19xoV6aMwZt6iX/C30RWfg= github.com/sclevine/agouti v3.0.0+incompatible/go.mod h1:b4WX9W9L1sfQKXeJf1mUTLZKJ48R1S7H23Ji7oFO5Bw= github.com/sclevine/spec v1.2.0/go.mod h1:W4J29eT/Kzv7/b9IWLB055Z+qvVC9vt0Arko24q7p+U= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= @@ -2199,6 +2212,8 @@ golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9sn golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.3.0 h1:qoo4akIqOcDME5bhc/NgxUdovd6BSS2uMsVjB56q1xI= +golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= diff --git a/weed/replication/sink/filersink/fetch_write.go b/weed/replication/sink/filersink/fetch_write.go index 7d526513e..d39589047 100644 --- a/weed/replication/sink/filersink/fetch_write.go +++ b/weed/replication/sink/filersink/fetch_write.go @@ -2,9 +2,11 @@ package filersink import ( "fmt" - "sync" - + "github.com/schollz/progressbar/v3" "github.com/seaweedfs/seaweedfs/weed/util" + "os" + "path/filepath" + "sync" "google.golang.org/grpc" @@ -19,6 +21,20 @@ func (fs *FilerSink) replicateChunks(sourceChunks []*filer_pb.FileChunk, path st return } + // a simple progress bar. Not ideal. Fix me. + var bar *progressbar.ProgressBar + if len(sourceChunks) > 1 { + name := filepath.Base(path) + bar = progressbar.NewOptions64(int64(len(sourceChunks)), + progressbar.OptionClearOnFinish(), + progressbar.OptionOnCompletion(func() { + fmt.Fprint(os.Stderr, "\n") + }), + progressbar.OptionFullWidth(), + progressbar.OptionSetDescription(name), + ) + } + replicatedChunks = make([]*filer_pb.FileChunk, len(sourceChunks)) var wg sync.WaitGroup @@ -34,6 +50,9 @@ func (fs *FilerSink) replicateChunks(sourceChunks []*filer_pb.FileChunk, path st return e } replicatedChunks[index] = replicatedChunk + if bar != nil { + bar.Add(1) + } err = nil return nil }) From d1797deccb817ae89edda016d192bcf45f80ea32 Mon Sep 17 00:00:00 2001 From: chrislu Date: Wed, 21 Dec 2022 14:15:11 -0800 Subject: [PATCH 18/25] remove duplicated logic of removing from writableChunks --- weed/mount/page_writer/upload_pipeline.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/weed/mount/page_writer/upload_pipeline.go b/weed/mount/page_writer/upload_pipeline.go index bce062a6e..252dddc06 100644 --- a/weed/mount/page_writer/upload_pipeline.go +++ b/weed/mount/page_writer/upload_pipeline.go @@ -73,9 +73,7 @@ func (up *UploadPipeline) SaveDataAt(p []byte, off int64, isSequential bool) (n fullness = chunkFullness } } - fullWritableChunk := up.writableChunks[fullestChunkIndex] - delete(up.writableChunks, fullestChunkIndex) - up.moveToSealed(fullWritableChunk, fullestChunkIndex) + up.moveToSealed(up.writableChunks[fullestChunkIndex], fullestChunkIndex) // fmt.Printf("flush chunk %d with %d bytes written\n", logicChunkIndex, fullness) } if isSequential && From 77b7c9f7ccce5ac9e4b0cb22b6018e5758a4c380 Mon Sep 17 00:00:00 2001 From: chrislu Date: Wed, 21 Dec 2022 14:15:35 -0800 Subject: [PATCH 19/25] lock and unlock chunks for read --- weed/mount/page_writer/upload_pipeline_lock.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/weed/mount/page_writer/upload_pipeline_lock.go b/weed/mount/page_writer/upload_pipeline_lock.go index 43b04e01d..fec61b532 100644 --- a/weed/mount/page_writer/upload_pipeline_lock.go +++ b/weed/mount/page_writer/upload_pipeline_lock.go @@ -10,6 +10,8 @@ func (up *UploadPipeline) LockForRead(startOffset, stopOffset int64) { if stopOffset%up.ChunkSize > 0 { stopLogicChunkIndex += 1 } + up.chunksLock.Lock() + defer up.chunksLock.Unlock() for i := startLogicChunkIndex; i < stopLogicChunkIndex; i++ { if count, found := up.activeReadChunks[i]; found { up.activeReadChunks[i] = count + 1 @@ -25,6 +27,8 @@ func (up *UploadPipeline) UnlockForRead(startOffset, stopOffset int64) { if stopOffset%up.ChunkSize > 0 { stopLogicChunkIndex += 1 } + up.chunksLock.Lock() + defer up.chunksLock.Unlock() for i := startLogicChunkIndex; i < stopLogicChunkIndex; i++ { if count, found := up.activeReadChunks[i]; found { if count == 1 { From 569bc7b54df308975169151b2ced7ab5339e9c5a Mon Sep 17 00:00:00 2001 From: Konstantin Lebedev <9497591+kmlebedev@users.noreply.github.com> Date: Thu, 22 Dec 2022 04:54:17 +0500 Subject: [PATCH 20/25] fs.verify needles status of file (#4060) * fs.verify needles status of file * fix parse args * refactor logging * set modifyTimeAgoAtSec --- weed/shell/command_fs_verify.go | 177 ++++++++++++++++++++++++++++++++ 1 file changed, 177 insertions(+) create mode 100644 weed/shell/command_fs_verify.go diff --git a/weed/shell/command_fs_verify.go b/weed/shell/command_fs_verify.go new file mode 100644 index 000000000..11cc64d78 --- /dev/null +++ b/weed/shell/command_fs_verify.go @@ -0,0 +1,177 @@ +package shell + +import ( + "context" + "flag" + "fmt" + "github.com/seaweedfs/seaweedfs/weed/filer" + "github.com/seaweedfs/seaweedfs/weed/operation" + "github.com/seaweedfs/seaweedfs/weed/pb" + "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" + "github.com/seaweedfs/seaweedfs/weed/pb/master_pb" + "github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb" + "github.com/seaweedfs/seaweedfs/weed/storage" + "github.com/seaweedfs/seaweedfs/weed/util" + "io" + "math" + "strings" + "time" +) + +func init() { + Commands = append(Commands, &commandFsVerify{}) +} + +type commandFsVerify struct { + env *CommandEnv + volumeIds map[uint32][]pb.ServerAddress + verbose *bool + modifyTimeAgoAtSec int64 + writer io.Writer +} + +func (c *commandFsVerify) Name() string { + return "fs.verify" +} + +func (c *commandFsVerify) Help() string { + return `recursively verify all files under a directory + + fs.verify [-v] [-modifyTimeAgo 1h] /buckets/dir + +` +} + +func (c *commandFsVerify) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) { + c.env = commandEnv + c.writer = writer + + fsVerifyCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError) + c.verbose = fsVerifyCommand.Bool("v", false, "print out each processed files") + modifyTimeAgo := fsVerifyCommand.Duration("modifyTimeAgo", 0, "only include files after this modify time to verify") + + if err = fsVerifyCommand.Parse(args); err != nil { + return err + } + + path, parseErr := commandEnv.parseUrl(findInputDirectory(fsVerifyCommand.Args())) + if parseErr != nil { + return parseErr + } + + c.modifyTimeAgoAtSec = int64(modifyTimeAgo.Seconds()) + + if err := c.collectVolumeIds(); err != nil { + return parseErr + } + + fCount, eConut, terr := c.verifyTraverseBfs(path) + + if terr == nil { + fmt.Fprintf(writer, "verified %d files, error %d files \n", fCount, eConut) + } + + return terr + +} + +func (c *commandFsVerify) collectVolumeIds() error { + c.volumeIds = make(map[uint32][]pb.ServerAddress) + topologyInfo, _, err := collectTopologyInfo(c.env, 0) + if err != nil { + return err + } + eachDataNode(topologyInfo, func(dc string, rack RackId, nodeInfo *master_pb.DataNodeInfo) { + for _, diskInfo := range nodeInfo.DiskInfos { + for _, vi := range diskInfo.VolumeInfos { + c.volumeIds[vi.Id] = append(c.volumeIds[vi.Id], pb.NewServerAddressFromDataNode(nodeInfo)) + } + } + }) + return nil +} + +func (c *commandFsVerify) verifyEntry(fileId *filer_pb.FileId, volumeServer *pb.ServerAddress) error { + err := operation.WithVolumeServerClient(false, *volumeServer, c.env.option.GrpcDialOption, + func(client volume_server_pb.VolumeServerClient) error { + _, err := client.VolumeNeedleStatus(context.Background(), + &volume_server_pb.VolumeNeedleStatusRequest{ + VolumeId: fileId.VolumeId, + NeedleId: fileId.FileKey}) + return err + }, + ) + if err != nil && !strings.Contains(err.Error(), storage.ErrorDeleted.Error()) { + return err + } + if *c.verbose { + fmt.Fprintf(c.writer, ".") + } + return nil +} + +type ItemEntry struct { + chunks []*filer_pb.FileChunk + path util.FullPath +} + +func (c *commandFsVerify) verifyTraverseBfs(path string) (fileCount int64, errCount int64, err error) { + timeNowAtSec := time.Now().Unix() + return fileCount, errCount, doTraverseBfsAndSaving(c.env, nil, path, false, + func(entry *filer_pb.FullEntry, outputChan chan interface{}) (err error) { + if c.modifyTimeAgoAtSec > 0 { + if entry.Entry.Attributes != nil && c.modifyTimeAgoAtSec < timeNowAtSec-entry.Entry.Attributes.Mtime { + return nil + } + } + dataChunks, manifestChunks, resolveErr := filer.ResolveChunkManifest(filer.LookupFn(c.env), entry.Entry.GetChunks(), 0, math.MaxInt64) + if resolveErr != nil { + return fmt.Errorf("failed to ResolveChunkManifest: %+v", resolveErr) + } + dataChunks = append(dataChunks, manifestChunks...) + if len(dataChunks) > 0 { + outputChan <- &ItemEntry{ + chunks: dataChunks, + path: util.NewFullPath(entry.Dir, entry.Entry.Name), + } + } + return nil + }, + func(outputChan chan interface{}) { + for itemEntry := range outputChan { + i := itemEntry.(*ItemEntry) + fileMsg := fmt.Sprintf("file:%s needle status ", i.path) + if *c.verbose { + fmt.Fprintf(c.writer, fileMsg) + fileMsg = "" + } + for _, chunk := range i.chunks { + if volumeIds, ok := c.volumeIds[chunk.Fid.VolumeId]; ok { + for _, volumeServer := range volumeIds { + if err = c.verifyEntry(chunk.Fid, &volumeServer); err != nil { + fmt.Fprintf(c.writer, "%sfailed verify %d:%d: %+v\n", + fileMsg, chunk.Fid.VolumeId, chunk.Fid.FileKey, err) + break + } + } + } else { + err = fmt.Errorf("volumeId %d not found", chunk.Fid.VolumeId) + fmt.Fprintf(c.writer, "%sfailed verify chunk %d:%d: %+v\n", + fileMsg, chunk.Fid.VolumeId, chunk.Fid.FileKey, err) + break + } + } + + if err != nil { + errCount++ + continue + } + + if *c.verbose { + fmt.Fprintf(c.writer, " verifed\n") + } + fileCount++ + } + }) + +} From 3901f2da4dd354ab9973b36705068f075e1edbd0 Mon Sep 17 00:00:00 2001 From: fibbanachi <49879947+Programmeris@users.noreply.github.com> Date: Fri, 23 Dec 2022 18:36:15 +0300 Subject: [PATCH 21/25] Add logtostderr option in helm chart (#4080) --- k8s/helm_charts2/templates/filer-statefulset.yaml | 7 ++++++- k8s/helm_charts2/templates/master-statefulset.yaml | 7 ++++++- k8s/helm_charts2/templates/s3-deployment.yaml | 7 ++++++- k8s/helm_charts2/templates/volume-statefulset.yaml | 7 ++++++- 4 files changed, 24 insertions(+), 4 deletions(-) diff --git a/k8s/helm_charts2/templates/filer-statefulset.yaml b/k8s/helm_charts2/templates/filer-statefulset.yaml index 978cdeec8..ad1325609 100644 --- a/k8s/helm_charts2/templates/filer-statefulset.yaml +++ b/k8s/helm_charts2/templates/filer-statefulset.yaml @@ -101,7 +101,12 @@ spec: - "/bin/sh" - "-ec" - | - exec /usr/bin/weed -logdir=/logs \ + exec /usr/bin/weed \ + {{- if eq .Values.filer.logs.type "hostPath" }} + -logdir=/logs \ + {{- else }} + -logtostderr=true \ + {{- end }} {{- if .Values.filer.loggingOverrideLevel }} -v={{ .Values.filer.loggingOverrideLevel }} \ {{- else }} diff --git a/k8s/helm_charts2/templates/master-statefulset.yaml b/k8s/helm_charts2/templates/master-statefulset.yaml index c31962036..1584387fd 100644 --- a/k8s/helm_charts2/templates/master-statefulset.yaml +++ b/k8s/helm_charts2/templates/master-statefulset.yaml @@ -90,7 +90,12 @@ spec: - "/bin/sh" - "-ec" - | - exec /usr/bin/weed -logdir=/logs \ + exec /usr/bin/weed \ + {{- if eq .Values.master.logs.type "hostPath" }} + -logdir=/logs \ + {{- else }} + -logtostderr=true \ + {{- end }} {{- if .Values.master.loggingOverrideLevel }} -v={{ .Values.master.loggingOverrideLevel }} \ {{- else }} diff --git a/k8s/helm_charts2/templates/s3-deployment.yaml b/k8s/helm_charts2/templates/s3-deployment.yaml index c9e12e9cb..a13c0790a 100644 --- a/k8s/helm_charts2/templates/s3-deployment.yaml +++ b/k8s/helm_charts2/templates/s3-deployment.yaml @@ -72,7 +72,12 @@ spec: - "/bin/sh" - "-ec" - | - exec /usr/bin/weed -logdir=/logs \ + exec /usr/bin/weed \ + {{- if eq .Values.s3.logs.type "hostPath" }} + -logdir=/logs \ + {{- else }} + -logtostderr=true \ + {{- end }} {{- if .Values.s3.loggingOverrideLevel }} -v={{ .Values.s3.loggingOverrideLevel }} \ {{- else }} diff --git a/k8s/helm_charts2/templates/volume-statefulset.yaml b/k8s/helm_charts2/templates/volume-statefulset.yaml index 55431f477..86592252f 100644 --- a/k8s/helm_charts2/templates/volume-statefulset.yaml +++ b/k8s/helm_charts2/templates/volume-statefulset.yaml @@ -93,7 +93,12 @@ spec: - "/bin/sh" - "-ec" - | - exec /usr/bin/weed -logdir=/logs \ + exec /usr/bin/weed \ + {{- if eq .Values.volume.logs.type "hostPath" }} + -logdir=/logs \ + {{- else }} + -logtostderr=true \ + {{- end }} {{- if .Values.volume.loggingOverrideLevel }} -v={{ .Values.volume.loggingOverrideLevel }} \ {{- else }} From 5f993a5f20fd96f8c866d7e09e881156cb510415 Mon Sep 17 00:00:00 2001 From: Guo Lei Date: Mon, 26 Dec 2022 16:03:47 +0800 Subject: [PATCH 22/25] Fix ide warning (#4082) fix warning message from IDE "redundant type from array, slice, or map composite literal" --- weed/s3api/filer_multipart_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/weed/s3api/filer_multipart_test.go b/weed/s3api/filer_multipart_test.go index 128bde944..e76d903b8 100644 --- a/weed/s3api/filer_multipart_test.go +++ b/weed/s3api/filer_multipart_test.go @@ -57,19 +57,19 @@ func Test_findByPartNumber(t *testing.T) { } parts := []CompletedPart{ - CompletedPart{ + { ETag: "xxx", PartNumber: 1, }, - CompletedPart{ + { ETag: "lll", PartNumber: 1, }, - CompletedPart{ + { ETag: "yyy", PartNumber: 3, }, - CompletedPart{ + { ETag: "zzz", PartNumber: 5, }, From b2aa1a23ffd86674d794866c2879e2875e4f15af Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 26 Dec 2022 11:47:52 -0800 Subject: [PATCH 23/25] build(deps): bump github.com/arangodb/go-driver from 1.4.0 to 1.4.1 (#4083) --- go.mod | 6 ++---- go.sum | 13 ++----------- 2 files changed, 4 insertions(+), 15 deletions(-) diff --git a/go.mod b/go.mod index 01a1f0035..8debd915c 100644 --- a/go.mod +++ b/go.mod @@ -141,7 +141,7 @@ require ( require ( github.com/Jille/raft-grpc-transport v1.3.0 - github.com/arangodb/go-driver v1.4.0 + github.com/arangodb/go-driver v1.4.1 github.com/armon/go-metrics v0.4.1 github.com/fluent/fluent-logger-golang v1.9.0 github.com/google/flatbuffers v22.11.23+incompatible @@ -149,6 +149,7 @@ require ( github.com/hashicorp/raft v1.3.11 github.com/hashicorp/raft-boltdb/v2 v2.2.2 github.com/rabbitmq/amqp091-go v1.5.0 + github.com/schollz/progressbar/v3 v3.12.2 github.com/tikv/client-go/v2 v2.0.3 github.com/ydb-platform/ydb-go-sdk-auth-environ v0.1.2 github.com/ydb-platform/ydb-go-sdk/v3 v3.41.0 @@ -194,7 +195,6 @@ require ( github.com/jcmturner/rpc/v2 v2.0.3 // indirect github.com/jonboulle/clockwork v0.2.2 // indirect github.com/josharian/intern v1.0.0 // indirect - github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213 // indirect github.com/klauspost/cpuid/v2 v2.1.1 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-runewidth v0.0.14 // indirect @@ -214,8 +214,6 @@ require ( github.com/pingcap/log v1.1.1-0.20221015072633-39906604fb81 // indirect github.com/rivo/uniseg v0.4.3 // indirect github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd // indirect - github.com/schollz/progressbar v1.0.0 // indirect - github.com/schollz/progressbar/v3 v3.12.2 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/stathat/consistent v1.0.0 // indirect github.com/subosito/gotenv v1.4.1 // indirect diff --git a/go.sum b/go.sum index ddd76311f..0f02afc70 100644 --- a/go.sum +++ b/go.sum @@ -211,8 +211,8 @@ github.com/antihax/optional v0.0.0-20180407024304-ca021399b1a6/go.mod h1:V8iCPQY github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/apache/thrift v0.13.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= -github.com/arangodb/go-driver v1.4.0 h1:uNCbVYkr5ZP3hIVUP6wqjOVyhMYOL9NDmR762tIeYP0= -github.com/arangodb/go-driver v1.4.0/go.mod h1:5GAx3XvK72DJPhJgyjZOtYAGc4SpY7rZDb3LyhCvLcQ= +github.com/arangodb/go-driver v1.4.1 h1:Jg0N7XKxiKwjswmAcMCnefWmt81KJEqybqRAGJDRWlo= +github.com/arangodb/go-driver v1.4.1/go.mod h1:UTtaxTUMmyPWzKc2dsWWOZzZ3yM6aHWxn/eubGa3YmQ= github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e h1:Xg+hGrY2LcQBbxd0ZFdbGSyRKTYMZCfBbw/pMJFOk1g= github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e/go.mod h1:mq7Shfa/CaixoDxiyAAc5jZ6CVBAyPaNQCGS7mkj4Ho= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= @@ -453,7 +453,6 @@ github.com/containers/ocicrypt v1.1.2/go.mod h1:Dm55fwWm1YZAjYRaJ94z2mfZikIyIN4B github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= -github.com/coreos/go-iptables v0.4.3/go.mod h1:/mVI274lEDI2ns62jHCDnCyBF9Iwsmekav8Dbxlm1MU= github.com/coreos/go-iptables v0.4.5/go.mod h1:/mVI274lEDI2ns62jHCDnCyBF9Iwsmekav8Dbxlm1MU= github.com/coreos/go-iptables v0.5.0/go.mod h1:/mVI274lEDI2ns62jHCDnCyBF9Iwsmekav8Dbxlm1MU= github.com/coreos/go-iptables v0.6.0/go.mod h1:Qe8Bv2Xik5FyTXwgIbLAnv2sWSBmvWdFETJConOQ//Q= @@ -490,7 +489,6 @@ github.com/d4l3k/messagediff v1.2.1/go.mod h1:Oozbb1TVXFac9FtSIxHBMnBCq2qeH/2KkE github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dchest/uniuri v0.0.0-20160212164326-8902c56451e9/go.mod h1:GgB8SF9nRG+GqaDtLcwJZsQFhcogVCJ79j4EdT0c2V4= github.com/denisenkom/go-mssqldb v0.12.2/go.mod h1:lnIw1mZukFRZDJYQ0Pb833QS2IaC3l5HkEfra2LJ+sk= github.com/dennwc/varint v1.0.0/go.mod h1:hnItb35rvZvJrbTALZtY/iQfDs48JKRG1RPpgziApxA= github.com/denverdino/aliyungo v0.0.0-20190125010748-a747050bb1ba/go.mod h1:dV8lFg6daOBZbT6/BDGIz6Y3WFGn8juu6G+CQ6LHtl0= @@ -1092,7 +1090,6 @@ github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/X github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= -github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213 h1:qGQQKEcAR99REcMpsXCp3lJ03zYT1PkRd3kQGPn9GVg= github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213/go.mod h1:vNUNkEQ1e29fT/6vq2aBdFsgNPmy8qMdSay1npru+Sw= github.com/karlseguin/ccache/v2 v2.0.8 h1:lT38cE//uyf6KcFok0rlgXtGFBWxkI6h/qg4tbFyDnA= github.com/karlseguin/ccache/v2 v2.0.8/go.mod h1:2BDThcfQMf/c0jnZowt16eW405XIqZPavt+HoYEtcxQ= @@ -1196,8 +1193,6 @@ github.com/mattn/go-isatty v0.0.16 h1:bq3VjFmv/sOjHtdEhmkEV4x1AJtvUvOJ2PFAZ5+peK github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= -github.com/mattn/go-runewidth v0.0.7 h1:Ei8KR0497xHyKJPAv59M1dkC+rOZCMBJ+t3fZ+twI54= -github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU= github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mattn/go-shellwords v1.0.3/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vqg+NOMyg4B2o= @@ -1511,7 +1506,6 @@ github.com/rs/cors v1.8.2/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU= github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU= github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc= -github.com/rs/zerolog v1.19.0/go.mod h1:IzD0RJ65iWH0w97OQQebJEvTZYvsCUm9WVLWBQrJRjo= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd h1:CmH9+J6ZSsIjUK3dcGsnCnO41eRBOnY12zwkn5qVwgc= github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk= @@ -1522,8 +1516,6 @@ github.com/safchain/ethtool v0.0.0-20210803160452-9aa261dae9b1/go.mod h1:Z0q5wiB github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/scaleway/scaleway-sdk-go v1.0.0-beta.9/go.mod h1:fCa7OJZ/9DRTnOKmxvT6pn+LPWUptQAmHF/SBJUGEcg= -github.com/schollz/progressbar v1.0.0 h1:gbyFReLHDkZo8mxy/dLWMr+Mpb1MokGJ1FqCiqacjZM= -github.com/schollz/progressbar v1.0.0/go.mod h1:/l9I7PC3L3erOuz54ghIRKUEFcosiWfLvJv+Eq26UMs= github.com/schollz/progressbar/v3 v3.12.2 h1:yLqqqpQNMxGxHY8uEshRihaHWwa0rf0yb7/Zrpgq2C0= github.com/schollz/progressbar/v3 v3.12.2/go.mod h1:HFJYIYQQJX32UJdyoigUl19xoV6aMwZt6iX/C30RWfg= github.com/sclevine/agouti v3.0.0+incompatible/go.mod h1:b4WX9W9L1sfQKXeJf1mUTLZKJ48R1S7H23Ji7oFO5Bw= @@ -2269,7 +2261,6 @@ golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgw golang.org/x/tools v0.0.0-20190706070813-72ffa07ba3db/go.mod h1:jcCCGcm9btYwXyDqrUWc6MKQKKGJCWEQ3AfLSRIbEuI= golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190823170909-c4a336ef6a2f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20190828213141-aed303cbaa74/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190907020128-2ca718005c18/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= From bd176359cbb3c8b7c9137d3dfac3e9ecac8632f6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 26 Dec 2022 11:48:17 -0800 Subject: [PATCH 24/25] build(deps): bump github.com/prometheus/procfs from 0.8.0 to 0.9.0 (#4084) --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 8debd915c..376c0bc2b 100644 --- a/go.mod +++ b/go.mod @@ -80,7 +80,7 @@ require ( github.com/prometheus/client_golang v1.14.0 github.com/prometheus/client_model v0.3.0 // indirect github.com/prometheus/common v0.37.0 // indirect - github.com/prometheus/procfs v0.8.0 + github.com/prometheus/procfs v0.9.0 github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 // indirect github.com/seaweedfs/goexif v2.0.0+incompatible diff --git a/go.sum b/go.sum index 0f02afc70..008ccd25a 100644 --- a/go.sum +++ b/go.sum @@ -1478,8 +1478,8 @@ github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4O github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= -github.com/prometheus/procfs v0.8.0 h1:ODq8ZFEaYeCaZOJlZZdJA2AbQR98dSHSM1KW/You5mo= -github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= +github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJfhI= +github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY= github.com/prometheus/prometheus v0.35.0/go.mod h1:7HaLx5kEPKJ0GDgbODG0fZgXbQ8K/XjZNJXQmbmgQlY= github.com/prometheus/prometheus v0.37.0/go.mod h1:egARUgz+K93zwqsVIAneFlLZefyGOON44WyAp4Xqbbk= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= From 84492976c914cab5d084f4e242733906c20a3cc5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 26 Dec 2022 11:48:35 -0800 Subject: [PATCH 25/25] build(deps): bump github.com/aws/aws-sdk-go from 1.44.162 to 1.44.167 (#4085) --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 376c0bc2b..16f9dec67 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/Azure/azure-pipeline-go v0.2.3 github.com/Azure/azure-storage-blob-go v0.15.0 github.com/Shopify/sarama v1.37.2 - github.com/aws/aws-sdk-go v1.44.162 + github.com/aws/aws-sdk-go v1.44.167 github.com/beorn7/perks v1.0.1 // indirect github.com/bwmarrin/snowflake v0.3.0 github.com/cespare/xxhash/v2 v2.1.2 // indirect diff --git a/go.sum b/go.sum index 008ccd25a..84b26398a 100644 --- a/go.sum +++ b/go.sum @@ -239,8 +239,8 @@ github.com/aws/aws-sdk-go v1.43.11/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4 github.com/aws/aws-sdk-go v1.43.31/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= github.com/aws/aws-sdk-go v1.44.45/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= github.com/aws/aws-sdk-go v1.44.68/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo= -github.com/aws/aws-sdk-go v1.44.162 h1:hKAd+X+/BLxVMzH+4zKxbQcQQGrk2UhFX0OTu1Mhon8= -github.com/aws/aws-sdk-go v1.44.162/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= +github.com/aws/aws-sdk-go v1.44.167 h1:kQmBhGdZkQLU7AiHShSkBJ15zr8agy0QeaxXduvyp2E= +github.com/aws/aws-sdk-go v1.44.167/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= github.com/aws/aws-sdk-go-v2 v1.16.8 h1:gOe9UPR98XSf7oEJCcojYg+N2/jCRm4DdeIsP85pIyQ= github.com/aws/aws-sdk-go-v2 v1.16.8/go.mod h1:6CpKuLXg2w7If3ABZCl/qZ6rEgwtjZTn4eAf4RcEyuw=