From 731b3aadbeacc7754f5b91ce1b3f9b96eb428f3f Mon Sep 17 00:00:00 2001 From: "Dave St.Germain" Date: Fri, 10 May 2024 17:35:41 -0400 Subject: [PATCH 01/24] Add support for OpenBSD (#5578) Co-authored-by: Dave St.Germain --- weed/stats/disk_notsupported.go | 4 ++-- weed/stats/disk_openbsd.go | 25 +++++++++++++++++++++++++ 2 files changed, 27 insertions(+), 2 deletions(-) create mode 100644 weed/stats/disk_openbsd.go diff --git a/weed/stats/disk_notsupported.go b/weed/stats/disk_notsupported.go index 1da714c73..418164546 100644 --- a/weed/stats/disk_notsupported.go +++ b/weed/stats/disk_notsupported.go @@ -1,5 +1,5 @@ -//go:build openbsd || netbsd || plan9 || solaris -// +build openbsd netbsd plan9 solaris +//go:build netbsd || plan9 || solaris +// +build netbsd plan9 solaris package stats diff --git a/weed/stats/disk_openbsd.go b/weed/stats/disk_openbsd.go new file mode 100644 index 000000000..8224e626e --- /dev/null +++ b/weed/stats/disk_openbsd.go @@ -0,0 +1,25 @@ +//go:build openbsd +// +build openbsd + +package stats + +import ( + "syscall" + + "github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb" +) + +func fillInDiskStatus(disk *volume_server_pb.DiskStatus) { + fs := syscall.Statfs_t{} + err := syscall.Statfs(disk.Dir, &fs) + if err != nil { + return + } + disk.All = fs.F_blocks * uint64(fs.F_bsize) + disk.Free = fs.F_bfree * uint64(fs.F_bsize) + disk.Used = disk.All - disk.Free + disk.PercentFree = float32((float64(disk.Free) / float64(disk.All)) * 100) + disk.PercentUsed = float32((float64(disk.Used) / float64(disk.All)) * 100) + return +} + From d389c5b27e6ceef6fb623c0c2b8405c754d3ac5d Mon Sep 17 00:00:00 2001 From: Konstantin Lebedev <9497591+kmlebedev@users.noreply.github.com> Date: Sun, 12 May 2024 23:31:34 +0500 Subject: [PATCH 02/24] fix: recreate index include deleted files (#5579) * fix: recreate index include deleted files https://github.com/seaweedfs/seaweedfs/issues/5508 * fix: counting the number of files * fix: log --- weed/command/fix.go | 58 ++++++++++++++++++++++++------- weed/storage/needle_map_memory.go | 4 +-- weed/storage/volume_checking.go | 7 ++-- weed/storage/volume_vacuum.go | 26 +++++++------- 4 files changed, 64 insertions(+), 31 deletions(-) diff --git a/weed/command/fix.go b/weed/command/fix.go index b226a0b1a..4fb4ed88e 100644 --- a/weed/command/fix.go +++ b/weed/command/fix.go @@ -32,12 +32,15 @@ var cmdFix = &Command{ var ( fixVolumeCollection = cmdFix.Flag.String("collection", "", "an optional volume collection name, if specified only it will be processed") fixVolumeId = cmdFix.Flag.Int64("volumeId", 0, "an optional volume id, if not 0 (default) only it will be processed") + fixIncludeDeleted = cmdFix.Flag.Bool("includeDeleted", true, "include deleted entries in the index file") fixIgnoreError = cmdFix.Flag.Bool("ignoreError", false, "an optional, if true will be processed despite errors") ) type VolumeFileScanner4Fix struct { - version needle.Version - nm *needle_map.MemDb + version needle.Version + nm *needle_map.MemDb + nmDeleted *needle_map.MemDb + includeDeleted bool } func (scanner *VolumeFileScanner4Fix) VisitSuperBlock(superBlock super_block.SuperBlock) error { @@ -50,13 +53,20 @@ func (scanner *VolumeFileScanner4Fix) ReadNeedleBody() bool { } func (scanner *VolumeFileScanner4Fix) VisitNeedle(n *needle.Needle, offset int64, needleHeader, needleBody []byte) error { - glog.V(2).Infof("key %d offset %d size %d disk_size %d compressed %v", n.Id, offset, n.Size, n.DiskSize(scanner.version), n.IsCompressed()) + glog.V(2).Infof("key %v offset %d size %d disk_size %d compressed %v", n.Id, offset, n.Size, n.DiskSize(scanner.version), n.IsCompressed()) if n.Size.IsValid() { - pe := scanner.nm.Set(n.Id, types.ToOffset(offset), n.Size) - glog.V(2).Infof("saved %d with error %v", n.Size, pe) + if pe := scanner.nm.Set(n.Id, types.ToOffset(offset), n.Size); pe != nil { + return fmt.Errorf("saved %d with error %v", n.Size, pe) + } } else { - glog.V(2).Infof("skipping deleted file ...") - return scanner.nm.Delete(n.Id) + if scanner.includeDeleted { + if pe := scanner.nmDeleted.Set(n.Id, types.ToOffset(offset), types.TombstoneFileSize); pe != nil { + return fmt.Errorf("saved deleted %d with error %v", n.Size, pe) + } + } else { + glog.V(2).Infof("skipping deleted file ...") + return scanner.nm.Delete(n.Id) + } } return nil } @@ -109,21 +119,45 @@ func runFix(cmd *Command, args []string) bool { if *fixVolumeId != 0 && *fixVolumeId != volumeId { continue } - doFixOneVolume(basePath, baseFileName, collection, volumeId) + doFixOneVolume(basePath, baseFileName, collection, volumeId, *fixIncludeDeleted) } } return true } -func doFixOneVolume(basepath string, baseFileName string, collection string, volumeId int64) { +func SaveToIdx(scaner *VolumeFileScanner4Fix, idxName string) (ret error) { + idxFile, err := os.OpenFile(idxName, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644) + if err != nil { + return + } + defer func() { + idxFile.Close() + }() + + return scaner.nm.AscendingVisit(func(value needle_map.NeedleValue) error { + _, err := idxFile.Write(value.ToBytes()) + if scaner.includeDeleted && err == nil { + if deleted, ok := scaner.nmDeleted.Get(value.Key); ok { + _, err = idxFile.Write(deleted.ToBytes()) + } + } + return err + }) +} + +func doFixOneVolume(basepath string, baseFileName string, collection string, volumeId int64, fixIncludeDeleted bool) { indexFileName := path.Join(basepath, baseFileName+".idx") nm := needle_map.NewMemDb() + nmDeleted := needle_map.NewMemDb() defer nm.Close() + defer nmDeleted.Close() vid := needle.VolumeId(volumeId) scanner := &VolumeFileScanner4Fix{ - nm: nm, + nm: nm, + nmDeleted: nmDeleted, + includeDeleted: fixIncludeDeleted, } if err := storage.ScanVolumeFile(basepath, collection, vid, storage.NeedleMapInMemory, scanner); err != nil { @@ -135,12 +169,12 @@ func doFixOneVolume(basepath string, baseFileName string, collection string, vol } } - if err := nm.SaveToIdx(indexFileName); err != nil { - os.Remove(indexFileName) + if err := SaveToIdx(scanner, indexFileName); err != nil { err := fmt.Errorf("save to .idx File: %v", err) if *fixIgnoreError { glog.Error(err) } else { + os.Remove(indexFileName) glog.Fatal(err) } } diff --git a/weed/storage/needle_map_memory.go b/weed/storage/needle_map_memory.go index a2beb6c33..c75514a31 100644 --- a/weed/storage/needle_map_memory.go +++ b/weed/storage/needle_map_memory.go @@ -36,8 +36,8 @@ func LoadCompactNeedleMap(file *os.File) (*NeedleMap, error) { func doLoading(file *os.File, nm *NeedleMap) (*NeedleMap, error) { e := idx.WalkIndexFile(file, 0, func(key NeedleId, offset Offset, size Size) error { nm.MaybeSetMaxFileKey(key) - nm.FileCounter++ if !offset.IsZero() && size.IsValid() { + nm.FileCounter++ nm.FileByteCounter = nm.FileByteCounter + uint64(size) oldOffset, oldSize := nm.m.Set(NeedleId(key), offset, size) if !oldOffset.IsZero() && oldSize.IsValid() { @@ -51,7 +51,7 @@ func doLoading(file *os.File, nm *NeedleMap) (*NeedleMap, error) { } return nil }) - glog.V(1).Infof("max file key: %d for file: %s", nm.MaxFileKey(), file.Name()) + glog.V(1).Infof("max file key: %v count: %d deleted: %d for file: %s", nm.MaxFileKey(), nm.FileCount(), nm.DeletedCount(), file.Name()) return nm, e } diff --git a/weed/storage/volume_checking.go b/weed/storage/volume_checking.go index 0c5f154e8..f5ceffcce 100644 --- a/weed/storage/volume_checking.go +++ b/weed/storage/volume_checking.go @@ -109,9 +109,6 @@ func verifyNeedleIntegrity(datFile backend.BackendStorageFile, v needle.Version, return 0, fmt.Errorf("verifyNeedleIntegrity check %s entry offset %d size %d: %v", datFile.Name(), offset, size, err) } n.AppendAtNs = util.BytesToUint64(bytes) - if n.HasTtl() { - return n.AppendAtNs, nil - } fileTailOffset := offset + needle.GetActualSize(size, v) fileSize, _, err := datFile.GetStat() if err != nil { @@ -130,7 +127,7 @@ func verifyNeedleIntegrity(datFile backend.BackendStorageFile, v needle.Version, return n.AppendAtNs, fmt.Errorf("read data [%d,%d) : %v", offset, offset+int64(size), err) } if n.Id != key { - return n.AppendAtNs, fmt.Errorf("index key %#x does not match needle's Id %#x", key, n.Id) + return n.AppendAtNs, fmt.Errorf("index key %v does not match needle's Id %v", key, n.Id) } return n.AppendAtNs, err } @@ -147,7 +144,7 @@ func verifyDeletedNeedleIntegrity(datFile backend.BackendStorageFile, v needle.V return n.AppendAtNs, fmt.Errorf("read data [%d,%d) : %v", fileSize-size, size, err) } if n.Id != key { - return n.AppendAtNs, fmt.Errorf("index key %#x does not match needle's Id %#x", key, n.Id) + return n.AppendAtNs, fmt.Errorf("index key %v does not match needle's Id %v", key, n.Id) } return n.AppendAtNs, err } diff --git a/weed/storage/volume_vacuum.go b/weed/storage/volume_vacuum.go index c8098493d..6bbbde71d 100644 --- a/weed/storage/volume_vacuum.go +++ b/weed/storage/volume_vacuum.go @@ -487,19 +487,21 @@ func (v *Volume) copyDataBasedOnIndexFile(srcDatName, srcIdxName, dstDatName, da if err != nil { return err } - dstDatSize, _, err := dstDatBackend.GetStat() - if err != nil { - return err - } - if v.nm.ContentSize() > v.nm.DeletedSize() { - expectedContentSize := v.nm.ContentSize() - v.nm.DeletedSize() - if expectedContentSize > uint64(dstDatSize) { - return fmt.Errorf("volume %s unexpected new data size: %d does not match size of content minus deleted: %d", - v.Id.String(), dstDatSize, expectedContentSize) + if v.Ttl.String() == "" { + dstDatSize, _, err := dstDatBackend.GetStat() + if err != nil { + return err + } + if v.nm.ContentSize() > v.nm.DeletedSize() { + expectedContentSize := v.nm.ContentSize() - v.nm.DeletedSize() + if expectedContentSize > uint64(dstDatSize) { + return fmt.Errorf("volume %s unexpected new data size: %d does not match size of content minus deleted: %d", + v.Id.String(), dstDatSize, expectedContentSize) + } + } else { + glog.Warningf("volume %s content size: %d less deleted size: %d, new size: %d", + v.Id.String(), v.nm.ContentSize(), v.nm.DeletedSize(), dstDatSize) } - } else { - glog.Warningf("volume %s content size: %d less deleted size: %d, new size: %d", - v.Id.String(), v.nm.ContentSize(), v.nm.DeletedSize(), dstDatSize) } err = newNm.SaveToIdx(datIdxName) if err != nil { From b7d184c740024cc4087b85ede41ae35c027ab138 Mon Sep 17 00:00:00 2001 From: Gregor Tudan Date: Mon, 13 May 2024 16:20:14 +0200 Subject: [PATCH 03/24] Helm-Chart: Make MySQL credentials optional (#5583) --- k8s/charts/seaweedfs/templates/filer-statefulset.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/k8s/charts/seaweedfs/templates/filer-statefulset.yaml b/k8s/charts/seaweedfs/templates/filer-statefulset.yaml index 6da7ecbe0..30f2da537 100644 --- a/k8s/charts/seaweedfs/templates/filer-statefulset.yaml +++ b/k8s/charts/seaweedfs/templates/filer-statefulset.yaml @@ -84,11 +84,13 @@ spec: secretKeyRef: name: secret-seaweedfs-db key: user + optional: true - name: WEED_MYSQL_PASSWORD valueFrom: secretKeyRef: name: secret-seaweedfs-db key: password + optional: true - name: SEAWEEDFS_FULLNAME value: "{{ template "seaweedfs.name" . }}" {{- if .Values.filer.extraEnvironmentVars }} From 063683786a3c6e60a6361813e4eff501346924b3 Mon Sep 17 00:00:00 2001 From: Gregor Tudan Date: Mon, 13 May 2024 18:56:10 +0200 Subject: [PATCH 04/24] Replace hardcoded datasource uids in the grafana dashboard (#5590) --- .../seaweedfs-grafana-dashboard.json | 52 +++++++++++++------ 1 file changed, 36 insertions(+), 16 deletions(-) diff --git a/k8s/charts/seaweedfs/dashboards/seaweedfs-grafana-dashboard.json b/k8s/charts/seaweedfs/dashboards/seaweedfs-grafana-dashboard.json index da0a4973b..40e7807d6 100644 --- a/k8s/charts/seaweedfs/dashboards/seaweedfs-grafana-dashboard.json +++ b/k8s/charts/seaweedfs/dashboards/seaweedfs-grafana-dashboard.json @@ -4,6 +4,7 @@ { "builtIn": 1, "datasource": { + "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "enable": true, @@ -29,7 +30,7 @@ "collapsed": false, "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_PROMETHEUS}" }, "gridPos": { "h": 1, @@ -44,6 +45,7 @@ }, { "datasource": { + "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Whether master is leader or not", @@ -108,6 +110,7 @@ }, { "datasource": { + "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Count times leader changed", @@ -197,6 +200,7 @@ }, { "datasource": { + "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "description": "Heartbeats received from components", @@ -322,7 +326,7 @@ }, "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_PROMETHEUS}" }, "description": "Count replica placement mismatch", "fieldConfig": { @@ -396,7 +400,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "exemplar": true, @@ -462,7 +466,7 @@ }, "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_PROMETHEUS}" }, "description": "Total count of raft leaders", "fieldConfig": { @@ -615,7 +619,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_PROMETHEUS}" }, "editorMode": "code", "exemplar": true, @@ -637,7 +641,7 @@ "collapsed": false, "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_PROMETHEUS}" }, "gridPos": { "h": 1, @@ -656,6 +660,7 @@ "dashLength": 10, "dashes": false, "datasource": { + "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editable": true, @@ -754,6 +759,7 @@ "dashLength": 10, "dashes": false, "datasource": { + "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editable": true, @@ -858,6 +864,7 @@ "dashLength": 10, "dashes": false, "datasource": { + "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editable": true, @@ -962,6 +969,7 @@ "dashLength": 10, "dashes": false, "datasource": { + "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editable": true, @@ -1066,7 +1074,7 @@ "collapsed": false, "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_PROMETHEUS}" }, "gridPos": { "h": 1, @@ -1085,6 +1093,7 @@ "dashLength": 10, "dashes": false, "datasource": { + "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editable": true, @@ -1183,6 +1192,7 @@ "dashLength": 10, "dashes": false, "datasource": { + "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editable": true, @@ -1281,6 +1291,7 @@ "dashLength": 10, "dashes": false, "datasource": { + "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editable": true, @@ -1428,7 +1439,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_PROMETHEUS}" }, "exemplar": true, "expr": "histogram_quantile(0.99, sum(rate(SeaweedFS_s3_request_seconds_bucket[$__rate_interval])) by (le))", @@ -1443,7 +1454,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_PROMETHEUS}" }, "exemplar": true, "expr": "histogram_quantile(0.99, sum(rate(SeaweedFS_s3_request_seconds_bucket[$__rate_interval])) by (le, type, pod))", @@ -1541,7 +1552,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_PROMETHEUS}" }, "exemplar": true, "expr": "histogram_quantile(0.99, sum(rate(SeaweedFS_s3_request_seconds_bucket[$__rate_interval])) by (le, type, bucket))", @@ -1653,7 +1664,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_PROMETHEUS}" }, "expr": "sum (rate(SeaweedFS_s3_request_total[$__rate_interval])) by (type)", "format": "time_series", @@ -1703,6 +1714,7 @@ "dashLength": 10, "dashes": false, "datasource": { + "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editable": true, @@ -1832,7 +1844,7 @@ "collapsed": false, "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_PROMETHEUS}" }, "gridPos": { "h": 1, @@ -1947,6 +1959,7 @@ "dashLength": 10, "dashes": false, "datasource": { + "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editable": true, @@ -2082,7 +2095,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_PROMETHEUS}" }, "expr": "sum(SeaweedFS_volumeServer_volumes) by (collection, type)", "format": "time_series", @@ -2094,7 +2107,7 @@ { "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_PROMETHEUS}" }, "expr": "sum(SeaweedFS_volumeServer_volumes)", "format": "time_series", @@ -2141,6 +2154,7 @@ "dashLength": 10, "dashes": false, "datasource": { + "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fill": 1, @@ -2231,6 +2245,7 @@ "dashLength": 10, "dashes": false, "datasource": { + "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "fill": 1, @@ -2312,7 +2327,7 @@ "collapsed": false, "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_PROMETHEUS}" }, "gridPos": { "h": 1, @@ -2331,6 +2346,7 @@ "dashLength": 10, "dashes": false, "datasource": { + "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editable": true, @@ -2419,6 +2435,7 @@ "dashLength": 10, "dashes": false, "datasource": { + "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editable": true, @@ -2508,7 +2525,7 @@ "collapsed": false, "datasource": { "type": "prometheus", - "uid": "prometheus" + "uid": "${DS_PROMETHEUS}" }, "gridPos": { "h": 1, @@ -2527,6 +2544,7 @@ "dashLength": 10, "dashes": false, "datasource": { + "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editable": true, @@ -2648,6 +2666,7 @@ "dashLength": 10, "dashes": false, "datasource": { + "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editable": true, @@ -2738,6 +2757,7 @@ "dashLength": 10, "dashes": false, "datasource": { + "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, "editable": true, From fe3776c9dfefaa18fc82b70cb7a282822e015490 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 13 May 2024 10:05:01 -0700 Subject: [PATCH 05/24] chore(deps): bump github.com/aws/aws-sdk-go-v2/service/s3 from 1.53.1 to 1.53.2 (#5584) chore(deps): bump github.com/aws/aws-sdk-go-v2/service/s3 Bumps [github.com/aws/aws-sdk-go-v2/service/s3](https://github.com/aws/aws-sdk-go-v2) from 1.53.1 to 1.53.2. - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/service/s3/v1.53.1...service/s3/v1.53.2) --- updated-dependencies: - dependency-name: github.com/aws/aws-sdk-go-v2/service/s3 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index d925be77a..25a58229b 100644 --- a/go.mod +++ b/go.mod @@ -134,7 +134,7 @@ require ( github.com/aws/aws-sdk-go-v2 v1.26.1 github.com/aws/aws-sdk-go-v2/config v1.27.11 github.com/aws/aws-sdk-go-v2/credentials v1.17.11 - github.com/aws/aws-sdk-go-v2/service/s3 v1.53.1 + github.com/aws/aws-sdk-go-v2/service/s3 v1.53.2 github.com/cognusion/imaging v1.0.1 github.com/fluent/fluent-logger-golang v1.9.0 github.com/getsentry/sentry-go v0.27.0 diff --git a/go.sum b/go.sum index 9ab522cfd..f5c6ee203 100644 --- a/go.sum +++ b/go.sum @@ -168,8 +168,8 @@ github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.7 h1:ogRAwT1/g github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.7/go.mod h1:YCsIZhXfRPLFFCl5xxY+1T9RKzOKjCut+28JSX2DnAk= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.5 h1:f9RyWNtS8oH7cZlbn+/JNPpjUk5+5fLd5lM9M0i49Ys= github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.5/go.mod h1:h5CoMZV2VF297/VLhRhO1WF+XYWOzXo+4HsObA4HjBQ= -github.com/aws/aws-sdk-go-v2/service/s3 v1.53.1 h1:6cnno47Me9bRykw9AEv9zkXE+5or7jz8TsskTTccbgc= -github.com/aws/aws-sdk-go-v2/service/s3 v1.53.1/go.mod h1:qmdkIIAC+GCLASF7R2whgNrJADz0QZPX+Seiw/i4S3o= +github.com/aws/aws-sdk-go-v2/service/s3 v1.53.2 h1:rq2hglTQM3yHZvOPVMtNvLS5x6hijx7JvRDgKiTNDGQ= +github.com/aws/aws-sdk-go-v2/service/s3 v1.53.2/go.mod h1:qmdkIIAC+GCLASF7R2whgNrJADz0QZPX+Seiw/i4S3o= github.com/aws/aws-sdk-go-v2/service/sns v1.29.2 h1:kHm1SYs/NkxZpKINc4zOXOLJHVMzKtU4d7FlAMtDm50= github.com/aws/aws-sdk-go-v2/service/sns v1.29.2/go.mod h1:ZIs7/BaYel9NODoYa8PW39o15SFAXDEb4DxOG2It15U= github.com/aws/aws-sdk-go-v2/service/sqs v1.31.2 h1:A9ihuyTKpS8Z1ou/D4ETfOEFMyokA6JjRsgXWTiHvCk= From 25a7e18d3b4bcf504e844542c27ff07419b5aa89 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 13 May 2024 10:05:12 -0700 Subject: [PATCH 06/24] chore(deps): bump golang.org/x/image from 0.15.0 to 0.16.0 (#5585) Bumps [golang.org/x/image](https://github.com/golang/image) from 0.15.0 to 0.16.0. - [Commits](https://github.com/golang/image/compare/v0.15.0...v0.16.0) --- updated-dependencies: - dependency-name: golang.org/x/image dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 25a58229b..b6a73fd19 100644 --- a/go.mod +++ b/go.mod @@ -106,7 +106,7 @@ require ( gocloud.dev/pubsub/rabbitpubsub v0.37.0 golang.org/x/crypto v0.23.0 // indirect golang.org/x/exp v0.0.0-20240112132812-db7319d0e0e3 - golang.org/x/image v0.15.0 + golang.org/x/image v0.16.0 golang.org/x/net v0.25.0 golang.org/x/oauth2 v0.19.0 // indirect golang.org/x/sys v0.20.0 diff --git a/go.sum b/go.sum index f5c6ee203..56790cf65 100644 --- a/go.sum +++ b/go.sum @@ -1088,8 +1088,8 @@ golang.org/x/exp v0.0.0-20240112132812-db7319d0e0e3 h1:hNQpMuAJe5CtcUqCXaWga3FHu golang.org/x/exp v0.0.0-20240112132812-db7319d0e0e3/go.mod h1:idGWGoKP1toJGkd5/ig9ZLuPcZBC3ewk7SzmH0uou08= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= -golang.org/x/image v0.15.0 h1:kOELfmgrmJlw4Cdb7g/QGuB3CvDrXbqEIww/pNtNBm8= -golang.org/x/image v0.15.0/go.mod h1:HUYqC05R2ZcZ3ejNQsIHQDQiwWM4JBqmm6MKANTp4LE= +golang.org/x/image v0.16.0 h1:9kloLAKhUufZhA12l5fwnx2NZW39/we1UhBesW433jw= +golang.org/x/image v0.16.0/go.mod h1:ugSZItdV4nOxyqp56HmXwH0Ry0nBCpjnZdpDaIHdoPs= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= From 4bc7ff604a4876e5ea0246d5558ca575c63174a3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 13 May 2024 10:05:21 -0700 Subject: [PATCH 07/24] chore(deps): bump cloud.google.com/go/pubsub from 1.37.0 to 1.38.0 (#5586) Bumps [cloud.google.com/go/pubsub](https://github.com/googleapis/google-cloud-go) from 1.37.0 to 1.38.0. - [Release notes](https://github.com/googleapis/google-cloud-go/releases) - [Changelog](https://github.com/googleapis/google-cloud-go/blob/main/CHANGES.md) - [Commits](https://github.com/googleapis/google-cloud-go/compare/pubsub/v1.37.0...pubsub/v1.38.0) --- updated-dependencies: - dependency-name: cloud.google.com/go/pubsub dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 6 +++--- go.sum | 24 ++++++++++++------------ 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/go.mod b/go.mod index b6a73fd19..2778526dd 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.22.0 require ( cloud.google.com/go v0.112.2 // indirect - cloud.google.com/go/pubsub v1.37.0 + cloud.google.com/go/pubsub v1.38.0 cloud.google.com/go/storage v1.40.0 github.com/Azure/azure-pipeline-go v0.2.3 github.com/Azure/azure-storage-blob-go v0.15.0 @@ -114,7 +114,7 @@ require ( golang.org/x/tools v0.21.0 golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect google.golang.org/api v0.177.0 - google.golang.org/genproto v0.0.0-20240311173647-c811ad7063a7 // indirect + google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda // indirect google.golang.org/grpc v1.63.2 google.golang.org/protobuf v1.34.0 gopkg.in/inf.v0 v0.9.1 // indirect @@ -329,7 +329,7 @@ require ( golang.org/x/sync v0.7.0 // indirect golang.org/x/term v0.20.0 // indirect golang.org/x/time v0.5.0 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20240314234333-6e1732d8331c // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240429193739-8cf5692501f6 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240429193739-8cf5692501f6 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect diff --git a/go.sum b/go.sum index 56790cf65..c31bbfdb1 100644 --- a/go.sum +++ b/go.sum @@ -32,14 +32,14 @@ cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7 cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= cloud.google.com/go/iam v1.1.7 h1:z4VHOhwKLF/+UYXAJDFwGtNF0b6gjsW1Pk9Ml0U/IoM= cloud.google.com/go/iam v1.1.7/go.mod h1:J4PMPg8TtyurAUvSmPj8FF3EDgY1SPRZxcUGrn7WXGA= -cloud.google.com/go/kms v1.15.7 h1:7caV9K3yIxvlQPAcaFffhlT7d1qpxjB1wHBtjWa13SM= -cloud.google.com/go/kms v1.15.7/go.mod h1:ub54lbsa6tDkUwnu4W7Yt1aAIFLnspgh0kPGToDukeI= +cloud.google.com/go/kms v1.15.8 h1:szIeDCowID8th2i8XE4uRev5PMxQFqW+JjwYxL9h6xs= +cloud.google.com/go/kms v1.15.8/go.mod h1:WoUHcDjD9pluCg7pNds131awnH429QGvRM3N/4MyoVs= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= -cloud.google.com/go/pubsub v1.37.0 h1:0uEEfaB1VIJzabPpwpZf44zWAKAme3zwKKxHk7vJQxQ= -cloud.google.com/go/pubsub v1.37.0/go.mod h1:YQOQr1uiUM092EXwKs56OPT650nwnawc+8/IjoUeGzQ= +cloud.google.com/go/pubsub v1.38.0 h1:J1OT7h51ifATIedjqk/uBNPh+1hkvUaH4VKbz4UuAsc= +cloud.google.com/go/pubsub v1.38.0/go.mod h1:IPMJSWSus/cu57UyR01Jqa/bNOQA+XnPF6Z4dKW4fAA= cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= @@ -991,8 +991,8 @@ github.com/zeebo/errs v1.3.0 h1:hmiaKqgYZzcVgRL1Vkc1Mn2914BbzB0IBxs+ebeutGs= github.com/zeebo/errs v1.3.0/go.mod h1:sgbWHsvVuTPHcqJJGQ1WhI5KbWlHYz+2+2C/LSEtCw4= github.com/zeebo/pcg v1.0.1 h1:lyqfGeWiv4ahac6ttHs+I5hwtH/+1mrhlCtVNQM2kHo= github.com/zeebo/pcg v1.0.1/go.mod h1:09F0S9iiKrwn9rlI5yjLkmrug154/YRW6KnnXVDM/l4= -go.einride.tech/aip v0.66.0 h1:XfV+NQX6L7EOYK11yoHHFtndeaWh3KbD9/cN/6iWEt8= -go.einride.tech/aip v0.66.0/go.mod h1:qAhMsfT7plxBX+Oy7Huol6YUvZ0ZzdUz26yZsQwfl1M= +go.einride.tech/aip v0.67.1 h1:d/4TW92OxXBngkSOwWS2CH5rez869KpKMaN44mdxkFI= +go.einride.tech/aip v0.67.1/go.mod h1:ZGX4/zKw8dcgzdLsrvpOOGxfxI2QSk12SlP7d6c0/XI= go.etcd.io/bbolt v1.3.8 h1:xs88BrvEv273UsB79e0hcVrlUWmS0a8upikMFhSyAtA= go.etcd.io/bbolt v1.3.8/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw= go.etcd.io/etcd/api/v3 v3.5.13 h1:8WXU2/NBge6AUF1K1gOexB6e07NgsN1hXK0rSTtgSp4= @@ -1018,8 +1018,8 @@ go.opentelemetry.io/otel v1.24.0 h1:0LAOdjNmQeSTzGBzduGe/rU4tZhMwL5rWgtp9Ku5Jfo= go.opentelemetry.io/otel v1.24.0/go.mod h1:W7b9Ozg4nkF5tWI5zsXkaKKDjdVjpD4oAt9Qi/MArHo= go.opentelemetry.io/otel/metric v1.24.0 h1:6EhoGWWK28x1fbpA4tYTOWBkPefTDQnb8WSGXlc88kI= go.opentelemetry.io/otel/metric v1.24.0/go.mod h1:VYhLe1rFfxuTXLgj4CBiyz+9WYBA8pNGJgDcSFRKBco= -go.opentelemetry.io/otel/sdk v1.22.0 h1:6coWHw9xw7EfClIC/+O31R8IY3/+EiRFHevmHafB2Gw= -go.opentelemetry.io/otel/sdk v1.22.0/go.mod h1:iu7luyVGYovrRpe2fmj3CVKouQNdTOkxtLzPvPz1DOc= +go.opentelemetry.io/otel/sdk v1.24.0 h1:YMPPDNymmQN3ZgczicBY3B6sf9n62Dlj9pWD3ucgoDw= +go.opentelemetry.io/otel/sdk v1.24.0/go.mod h1:KVrIYw6tEubO9E96HQpcmpTKDVn9gdv35HoYiQWGDFg= go.opentelemetry.io/otel/trace v1.24.0 h1:CsKnnL4dUAr/0llH9FKuc698G04IrpWV0MQA/Y1YELI= go.opentelemetry.io/otel/trace v1.24.0/go.mod h1:HPc3Xr/cOApsBI154IU0OI0HJexz+aw5uPdbs3UCjNU= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= @@ -1427,10 +1427,10 @@ google.golang.org/genproto v0.0.0-20200806141610-86f49bd18e98/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210903162649-d08c68adba83/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= google.golang.org/genproto v0.0.0-20220421151946-72621c1f0bd3/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= -google.golang.org/genproto v0.0.0-20240311173647-c811ad7063a7 h1:ImUcDPHjTrAqNhlOkSocDLfG9rrNHH7w7uoKWPaWZ8s= -google.golang.org/genproto v0.0.0-20240311173647-c811ad7063a7/go.mod h1:/3XmxOjePkvmKrHuBy4zNFw7IzxJXtAgdpXi8Ll990U= -google.golang.org/genproto/googleapis/api v0.0.0-20240314234333-6e1732d8331c h1:kaI7oewGK5YnVwj+Y+EJBO/YN1ht8iTL9XkFHtVZLsc= -google.golang.org/genproto/googleapis/api v0.0.0-20240314234333-6e1732d8331c/go.mod h1:VQW3tUculP/D4B+xVCo+VgSq8As6wA9ZjHl//pmk+6s= +google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda h1:wu/KJm9KJwpfHWhkkZGohVC6KRrc1oJNr4jwtQMOQXw= +google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda/go.mod h1:g2LLCvCeCSir/JJSWosk19BR4NVxGqHUC6rxIRsd7Aw= +google.golang.org/genproto/googleapis/api v0.0.0-20240429193739-8cf5692501f6 h1:DTJM0R8LECCgFeUwApvcEJHz85HLagW8uRENYxHh1ww= +google.golang.org/genproto/googleapis/api v0.0.0-20240429193739-8cf5692501f6/go.mod h1:10yRODfgim2/T8csjQsMPgZOMvtytXKTDRzH6HRGzRw= google.golang.org/genproto/googleapis/rpc v0.0.0-20240429193739-8cf5692501f6 h1:DujSIu+2tC9Ht0aPNA7jgj23Iq8Ewi5sgkQ++wdvonE= google.golang.org/genproto/googleapis/rpc v0.0.0-20240429193739-8cf5692501f6/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= From 66dfeb7ba2ec66a806727865e11ced689209d14b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 13 May 2024 10:05:31 -0700 Subject: [PATCH 08/24] chore(deps): bump github.com/shirou/gopsutil/v3 from 3.24.3 to 3.24.4 (#5587) Bumps [github.com/shirou/gopsutil/v3](https://github.com/shirou/gopsutil) from 3.24.3 to 3.24.4. - [Release notes](https://github.com/shirou/gopsutil/releases) - [Commits](https://github.com/shirou/gopsutil/compare/v3.24.3...v3.24.4) --- updated-dependencies: - dependency-name: github.com/shirou/gopsutil/v3 dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index 2778526dd..f1ebeb6f3 100644 --- a/go.mod +++ b/go.mod @@ -149,7 +149,7 @@ require ( github.com/rclone/rclone v1.66.0 github.com/rdleal/intervalst v1.4.0 github.com/schollz/progressbar/v3 v3.14.2 - github.com/shirou/gopsutil/v3 v3.24.3 + github.com/shirou/gopsutil/v3 v3.24.4 github.com/tikv/client-go/v2 v2.0.7 github.com/ydb-platform/ydb-go-sdk-auth-environ v0.2.0 github.com/ydb-platform/ydb-go-sdk/v3 v3.65.0 diff --git a/go.sum b/go.sum index c31bbfdb1..0b0e93520 100644 --- a/go.sum +++ b/go.sum @@ -844,8 +844,8 @@ github.com/segmentio/asm v1.1.3/go.mod h1:Ld3L4ZXGNcSLRg4JBsZ3//1+f/TjYl0Mzen/DQ github.com/segmentio/encoding v0.3.6 h1:E6lVLyDPseWEulBmCmAKPanDd3jiyGDo5gMcugCRwZQ= github.com/segmentio/encoding v0.3.6/go.mod h1:n0JeuIqEQrQoPDGsjo8UNd1iA0U8d8+oHAA4E3G3OxM= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= -github.com/shirou/gopsutil/v3 v3.24.3 h1:eoUGJSmdfLzJ3mxIhmOAhgKEKgQkeOwKpz1NbhVnuPE= -github.com/shirou/gopsutil/v3 v3.24.3/go.mod h1:JpND7O217xa72ewWz9zN2eIIkPWsDN/3pl0H8Qt0uwg= +github.com/shirou/gopsutil/v3 v3.24.4 h1:dEHgzZXt4LMNm+oYELpzl9YCqV65Yr/6SfrvgRBtXeU= +github.com/shirou/gopsutil/v3 v3.24.4/go.mod h1:lTd2mdiOspcqLgAnr9/nGi71NkeMpWKdmhuxm9GusH8= github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM= github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ= github.com/shoenig/test v0.6.4 h1:kVTaSd7WLz5WZ2IaoM0RSzRsUD+m8wRR+5qvntpn4LU= @@ -1266,7 +1266,7 @@ golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y= golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= From c23e9f96a5e172537534e0357da234eb0e517941 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 13 May 2024 10:05:50 -0700 Subject: [PATCH 09/24] chore(deps): bump github.com/ydb-platform/ydb-go-sdk/v3 from 3.65.0 to 3.66.3 (#5588) chore(deps): bump github.com/ydb-platform/ydb-go-sdk/v3 Bumps [github.com/ydb-platform/ydb-go-sdk/v3](https://github.com/ydb-platform/ydb-go-sdk) from 3.65.0 to 3.66.3. - [Release notes](https://github.com/ydb-platform/ydb-go-sdk/releases) - [Changelog](https://github.com/ydb-platform/ydb-go-sdk/blob/master/CHANGELOG.md) - [Commits](https://github.com/ydb-platform/ydb-go-sdk/compare/v3.65.0...v3.66.3) --- updated-dependencies: - dependency-name: github.com/ydb-platform/ydb-go-sdk/v3 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index f1ebeb6f3..b2c2ae7de 100644 --- a/go.mod +++ b/go.mod @@ -152,7 +152,7 @@ require ( github.com/shirou/gopsutil/v3 v3.24.4 github.com/tikv/client-go/v2 v2.0.7 github.com/ydb-platform/ydb-go-sdk-auth-environ v0.2.0 - github.com/ydb-platform/ydb-go-sdk/v3 v3.65.0 + github.com/ydb-platform/ydb-go-sdk/v3 v3.66.3 go.etcd.io/etcd/client/pkg/v3 v3.5.13 go.uber.org/atomic v1.11.0 google.golang.org/grpc/security/advancedtls v0.0.0-20220622233350-5cdb09fa29c1 @@ -309,7 +309,7 @@ require ( github.com/twmb/murmur3 v1.1.3 // indirect github.com/xanzy/ssh-agent v0.3.3 // indirect github.com/yandex-cloud/go-genproto v0.0.0-20211115083454-9ca41db5ed9e // indirect - github.com/ydb-platform/ydb-go-genproto v0.0.0-20240126124512-dbb0e1720dbf // indirect + github.com/ydb-platform/ydb-go-genproto v0.0.0-20240316140903-4a47abca1cca // indirect github.com/ydb-platform/ydb-go-yc v0.10.2 // indirect github.com/ydb-platform/ydb-go-yc-metadata v0.5.2 // indirect github.com/yunify/qingstor-sdk-go/v3 v3.2.0 // indirect diff --git a/go.sum b/go.sum index 0b0e93520..b3bb772ae 100644 --- a/go.sum +++ b/go.sum @@ -958,14 +958,14 @@ github.com/yandex-cloud/go-genproto v0.0.0-20211115083454-9ca41db5ed9e h1:9LPdmD github.com/yandex-cloud/go-genproto v0.0.0-20211115083454-9ca41db5ed9e/go.mod h1:HEUYX/p8966tMUHHT+TsS0hF/Ca/NYwqprC5WXSDMfE= github.com/ydb-platform/ydb-go-genproto v0.0.0-20220203104745-929cf9c248bc/go.mod h1:cc138nptTn9eKptCQl/grxP6pBKpo/bnXDiOxuVZtps= github.com/ydb-platform/ydb-go-genproto v0.0.0-20230528143953-42c825ace222/go.mod h1:Er+FePu1dNUieD+XTMDduGpQuCPssK5Q4BjF+IIXJ3I= -github.com/ydb-platform/ydb-go-genproto v0.0.0-20240126124512-dbb0e1720dbf h1:ckwNHVo4bv2tqNkgx3W3HANh3ta1j6TR5qw08J1A7Tw= -github.com/ydb-platform/ydb-go-genproto v0.0.0-20240126124512-dbb0e1720dbf/go.mod h1:Er+FePu1dNUieD+XTMDduGpQuCPssK5Q4BjF+IIXJ3I= +github.com/ydb-platform/ydb-go-genproto v0.0.0-20240316140903-4a47abca1cca h1:PliQWLwi2gTSOk7QyYQ9GfjvvikmibLWmaplKHy+kfo= +github.com/ydb-platform/ydb-go-genproto v0.0.0-20240316140903-4a47abca1cca/go.mod h1:Er+FePu1dNUieD+XTMDduGpQuCPssK5Q4BjF+IIXJ3I= github.com/ydb-platform/ydb-go-sdk-auth-environ v0.2.0 h1:IG5bPd+Lqyc+zsw2kmxqfGLkaDHuAEnWX63/8RBBiA4= github.com/ydb-platform/ydb-go-sdk-auth-environ v0.2.0/go.mod h1:l6lZ+osdQOjDRBgRA4PQ06BuvmXN2neYjnRw8rCfd7s= github.com/ydb-platform/ydb-go-sdk/v3 v3.25.3/go.mod h1:PFizF/vJsdAgEwjK3DVSBD52kdmRkWfSIS2q2pA+e88= github.com/ydb-platform/ydb-go-sdk/v3 v3.47.3/go.mod h1:bWnOIcUHd7+Sl7DN+yhyY1H/I61z53GczvwJgXMgvj0= -github.com/ydb-platform/ydb-go-sdk/v3 v3.65.0 h1:JHnWtMHGttTRgIDPB5waC9WYcaEsL4x4LWfrs/zd6jQ= -github.com/ydb-platform/ydb-go-sdk/v3 v3.65.0/go.mod h1:hJnZV1xFlpwNeH5Hy3N2TlLYl18OmDP5P1puEsRDNrY= +github.com/ydb-platform/ydb-go-sdk/v3 v3.66.3 h1:XeY8gUh18MNJW0+TmPx8rl+vpnWwwiqgII27JdPmgJA= +github.com/ydb-platform/ydb-go-sdk/v3 v3.66.3/go.mod h1:hGX4CijskNnUTRgLlqMvZdrBQc1ALZgAnKHytF5nmj4= github.com/ydb-platform/ydb-go-yc v0.10.2 h1:RAHy6g7ncxk1y0N4oS2MwYXLATqRqKBI6DYXuxpV2wo= github.com/ydb-platform/ydb-go-yc v0.10.2/go.mod h1:U1dX3LJy6zADId2DciCXlgrU/vphK1+CQzaefKq21dQ= github.com/ydb-platform/ydb-go-yc-metadata v0.5.2 h1:nMtixUijP0Z7iHJNT9fOL+dbmEzZxqU6Xk87ll7hqXg= From 3f3f0ec17f959bb5604f0acb10a72a0f9e6c1aac Mon Sep 17 00:00:00 2001 From: Gregor Tudan Date: Mon, 13 May 2024 19:16:58 +0200 Subject: [PATCH 10/24] Allow selecting the namespace visualized by the grafana dashboard (#5591) --- .../seaweedfs-grafana-dashboard.json | 114 +++++++++++------- 1 file changed, 71 insertions(+), 43 deletions(-) diff --git a/k8s/charts/seaweedfs/dashboards/seaweedfs-grafana-dashboard.json b/k8s/charts/seaweedfs/dashboards/seaweedfs-grafana-dashboard.json index 40e7807d6..47f5212b3 100644 --- a/k8s/charts/seaweedfs/dashboards/seaweedfs-grafana-dashboard.json +++ b/k8s/charts/seaweedfs/dashboards/seaweedfs-grafana-dashboard.json @@ -95,7 +95,7 @@ "targets": [ { "exemplar": true, - "expr": "sum by (pod) (SeaweedFS_master_is_leader{job=\"seaweedfs-master\"})", + "expr": "sum by (pod) (SeaweedFS_master_is_leader{job=\"seaweedfs-master\", namespace=\"$NAMESPACE\"})", "format": "time_series", "hide": false, "interval": "", @@ -184,7 +184,7 @@ "targets": [ { "exemplar": true, - "expr": "sum by (pod) (SeaweedFS_master_leader_changes{job=\"seaweedfs-master\", type=~\".+\"})", + "expr": "sum by (pod) (SeaweedFS_master_leader_changes{job=\"seaweedfs-master\", type=~\".+\", namespace=\"$NAMESPACE\"})", "format": "time_series", "hide": false, "instant": false, @@ -274,7 +274,7 @@ "targets": [ { "exemplar": true, - "expr": "sum by (type) (increase(SeaweedFS_master_received_heartbeats{job=\"seaweedfs-master\"}[$__rate_interval]))", + "expr": "sum by (type) (increase(SeaweedFS_master_received_heartbeats{job=\"seaweedfs-master\", namespace=\"$NAMESPACE\"}[$__rate_interval]))", "format": "time_series", "hide": false, "interval": "", @@ -404,7 +404,7 @@ }, "editorMode": "code", "exemplar": true, - "expr": "sum (SeaweedFS_master_replica_placement_mismatch{job=\"seaweedfs-master\"} > 0) by (pod)", + "expr": "sum (SeaweedFS_master_replica_placement_mismatch{job=\"seaweedfs-master\", namespace=\"$NAMESPACE\"} > 0) by (pod)", "format": "time_series", "hide": false, "instant": false, @@ -539,7 +539,7 @@ "targets": [ { "exemplar": true, - "expr": "sum (SeaweedFS_master_is_leader{job=\"seaweedfs-master\"})", + "expr": "sum (SeaweedFS_master_is_leader{job=\"seaweedfs-master\", namespace=\"$NAMESPACE\"})", "format": "time_series", "hide": false, "interval": "", @@ -623,7 +623,7 @@ }, "editorMode": "code", "exemplar": true, - "expr": "SeaweedFS_master_admin_lock{job=\"seaweedfs-master\"}", + "expr": "SeaweedFS_master_admin_lock{job=\"seaweedfs-master\", namespace=\"$NAMESPACE\"}", "format": "time_series", "hide": false, "interval": "", @@ -703,7 +703,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.90, sum(rate(SeaweedFS_filer_request_seconds_bucket[$__rate_interval])) by (le))", + "expr": "histogram_quantile(0.90, sum(rate(SeaweedFS_filer_request_seconds_bucket{namespace=\"$NAMESPACE\"}[$__rate_interval])) by (le))", "format": "time_series", "hide": false, "intervalFactor": 2, @@ -712,7 +712,7 @@ "step": 60 }, { - "expr": "histogram_quantile(0.90, sum(rate(SeaweedFS_filer_request_seconds_bucket[$__rate_interval])) by (le, type))", + "expr": "histogram_quantile(0.90, sum(rate(SeaweedFS_filer_request_seconds_bucket{namespace=\"$NAMESPACE\"}[$__rate_interval])) by (le, type))", "format": "time_series", "hide": false, "intervalFactor": 2, @@ -802,7 +802,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.95, sum(rate(SeaweedFS_filer_request_seconds_bucket[$__rate_interval])) by (le))", + "expr": "histogram_quantile(0.95, sum(rate(SeaweedFS_filer_request_seconds_bucket{namespace=\"$NAMESPACE\"}[$__rate_interval])) by (le))", "format": "time_series", "hide": false, "intervalFactor": 2, @@ -811,7 +811,7 @@ "step": 60 }, { - "expr": "histogram_quantile(0.95, sum(rate(SeaweedFS_filer_request_seconds_bucket[$__rate_interval])) by (le, type))", + "expr": "histogram_quantile(0.95, sum(rate(SeaweedFS_filer_request_seconds_bucket{namespace=\"$NAMESPACE\"}[$__rate_interval])) by (le, type))", "format": "time_series", "hide": false, "intervalFactor": 2, @@ -907,7 +907,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(SeaweedFS_filer_request_seconds_bucket[$__rate_interval])) by (le))", + "expr": "histogram_quantile(0.99, sum(rate(SeaweedFS_filer_request_seconds_bucket{namespace=\"$NAMESPACE\"}[$__rate_interval])) by (le))", "format": "time_series", "hide": false, "intervalFactor": 2, @@ -916,7 +916,7 @@ "step": 60 }, { - "expr": "histogram_quantile(0.99, sum(rate(SeaweedFS_filer_request_seconds_bucket[$__rate_interval])) by (le, type))", + "expr": "histogram_quantile(0.99, sum(rate(SeaweedFS_filer_request_seconds_bucket{namespace=\"$NAMESPACE\"}[$__rate_interval])) by (le, type))", "format": "time_series", "hide": false, "intervalFactor": 2, @@ -1027,7 +1027,7 @@ "targets": [ { "exemplar": true, - "expr": "sum by (type) (rate(SeaweedFS_filer_request_total[$__rate_interval]))", + "expr": "sum by (type) (rate(SeaweedFS_filer_request_total{namespace=\"$NAMESPACE\"}[$__rate_interval]))", "format": "time_series", "interval": "", "intervalFactor": 2, @@ -1136,7 +1136,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.90, sum(rate(SeaweedFS_s3_request_seconds_bucket[$__rate_interval])) by (le))", + "expr": "histogram_quantile(0.90, sum(rate(SeaweedFS_s3_request_seconds_bucket{namespace=\"$NAMESPACE\"}[$__rate_interval])) by (le))", "format": "time_series", "hide": false, "intervalFactor": 2, @@ -1145,7 +1145,7 @@ "step": 60 }, { - "expr": "histogram_quantile(0.90, sum(rate(SeaweedFS_s3_request_seconds_bucket[$__rate_interval])) by (le, type))", + "expr": "histogram_quantile(0.90, sum(rate(SeaweedFS_s3_request_seconds_bucket{namespace=\"$NAMESPACE\"}[$__rate_interval])) by (le, type))", "format": "time_series", "hide": false, "intervalFactor": 2, @@ -1235,7 +1235,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.95, sum(rate(SeaweedFS_s3_request_seconds_bucket[$__rate_interval])) by (le))", + "expr": "histogram_quantile(0.95, sum(rate(SeaweedFS_s3_request_seconds_bucket{namespace=\"$NAMESPACE\"}[$__rate_interval])) by (le))", "format": "time_series", "hide": false, "intervalFactor": 2, @@ -1244,7 +1244,7 @@ "step": 60 }, { - "expr": "histogram_quantile(0.95, sum(rate(SeaweedFS_s3_request_seconds_bucket[$__rate_interval])) by (le, type))", + "expr": "histogram_quantile(0.95, sum(rate(SeaweedFS_s3_request_seconds_bucket{namespace=\"$NAMESPACE\"}[$__rate_interval])) by (le, type))", "format": "time_series", "hide": false, "intervalFactor": 2, @@ -1334,7 +1334,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(SeaweedFS_s3_request_seconds_bucket[$__rate_interval])) by (le))", + "expr": "histogram_quantile(0.99, sum(rate(SeaweedFS_s3_request_seconds_bucket{namespace=\"$NAMESPACE\"}[$__rate_interval])) by (le))", "format": "time_series", "hide": false, "intervalFactor": 2, @@ -1344,7 +1344,7 @@ }, { "exemplar": true, - "expr": "histogram_quantile(0.99, sum(rate(SeaweedFS_s3_request_seconds_bucket[$__rate_interval])) by (le, type))", + "expr": "histogram_quantile(0.99, sum(rate(SeaweedFS_s3_request_seconds_bucket{namespace=\"$NAMESPACE\"}[$__rate_interval])) by (le, type))", "format": "time_series", "hide": false, "interval": "", @@ -1442,7 +1442,7 @@ "uid": "${DS_PROMETHEUS}" }, "exemplar": true, - "expr": "histogram_quantile(0.99, sum(rate(SeaweedFS_s3_request_seconds_bucket[$__rate_interval])) by (le))", + "expr": "histogram_quantile(0.99, sum(rate(SeaweedFS_s3_request_seconds_bucket{namespace=\"$NAMESPACE\"}[$__rate_interval])) by (le))", "format": "time_series", "hide": false, "interval": "", @@ -1457,7 +1457,7 @@ "uid": "${DS_PROMETHEUS}" }, "exemplar": true, - "expr": "histogram_quantile(0.99, sum(rate(SeaweedFS_s3_request_seconds_bucket[$__rate_interval])) by (le, type, pod))", + "expr": "histogram_quantile(0.99, sum(rate(SeaweedFS_s3_request_seconds_bucket{namespace=\"$NAMESPACE\"}[$__rate_interval])) by (le, type, pod))", "format": "time_series", "hide": false, "interval": "", @@ -1555,7 +1555,7 @@ "uid": "${DS_PROMETHEUS}" }, "exemplar": true, - "expr": "histogram_quantile(0.99, sum(rate(SeaweedFS_s3_request_seconds_bucket[$__rate_interval])) by (le, type, bucket))", + "expr": "histogram_quantile(0.99, sum(rate(SeaweedFS_s3_request_seconds_bucket{namespace=\"$NAMESPACE\"}[$__rate_interval])) by (le, type, bucket))", "format": "time_series", "hide": false, "interval": "", @@ -1666,7 +1666,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "sum (rate(SeaweedFS_s3_request_total[$__rate_interval])) by (type)", + "expr": "sum (rate(SeaweedFS_s3_request_total{namespace=\"$NAMESPACE\"}[$__rate_interval])) by (type)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{type}}", @@ -1771,7 +1771,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum by (type) (SeaweedFS_s3_request_total{type=~'PUT|COPY|POST|LIST'})*0.000005", + "expr": "sum by (type) (SeaweedFS_s3_request_total{type=~'PUT|COPY|POST|LIST', namespace=\"$NAMESPACE\"})*0.000005", "format": "time_series", "hide": false, "intervalFactor": 2, @@ -1780,7 +1780,7 @@ "step": 30 }, { - "expr": "sum (SeaweedFS_s3_request_total{type=~'PUT|COPY|POST|LIST'})*0.000005", + "expr": "sum (SeaweedFS_s3_request_total{type=~'PUT|COPY|POST|LIST', namespace=\"$NAMESPACE\"})*0.000005", "format": "time_series", "hide": false, "intervalFactor": 2, @@ -1789,7 +1789,7 @@ "step": 30 }, { - "expr": "sum (SeaweedFS_s3_request_total{type!~'PUT|COPY|POST|LIST'})*0.0000004", + "expr": "sum (SeaweedFS_s3_request_total{type!~'PUT|COPY|POST|LIST', namespace=\"$NAMESPACE\"})*0.0000004", "format": "time_series", "hide": false, "intervalFactor": 2, @@ -1797,7 +1797,7 @@ "refId": "B" }, { - "expr": "sum by (type) (SeaweedFS_s3_request_total{type!~'PUT|COPY|POST|LIST'})*0.0000004", + "expr": "sum by (type) (SeaweedFS_s3_request_total{type!~'PUT|COPY|POST|LIST', namespace=\"$NAMESPACE\"})*0.0000004", "format": "time_series", "hide": false, "intervalFactor": 2, @@ -1906,7 +1906,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(SeaweedFS_volumeServer_request_seconds_bucket[$__rate_interval])) by (le, exported_instance))", + "expr": "histogram_quantile(0.99, sum(rate(SeaweedFS_volumeServer_request_seconds_bucket{namespace=\"$NAMESPACE\"}[$__rate_interval])) by (le, exported_instance))", "format": "time_series", "hide": false, "intervalFactor": 2, @@ -1914,7 +1914,7 @@ "refId": "B" }, { - "expr": "histogram_quantile(0.99, sum(rate(SeaweedFS_volumeServer_request_seconds_bucket[$__rate_interval])) by (le))", + "expr": "histogram_quantile(0.99, sum(rate(SeaweedFS_volumeServer_request_seconds_bucket{namespace=\"$NAMESPACE\"}[$__rate_interval])) by (le))", "format": "time_series", "intervalFactor": 2, "legendFormat": "average", @@ -2008,7 +2008,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(SeaweedFS_volumeServer_request_total[$__rate_interval])) by (type)", + "expr": "sum(rate(SeaweedFS_volumeServer_request_total{namespace=\"$NAMESPACE\"}[$__rate_interval])) by (type)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{type}}", @@ -2097,7 +2097,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "sum(SeaweedFS_volumeServer_volumes) by (collection, type)", + "expr": "sum(SeaweedFS_volumeServer_volumes{namespace=\"$NAMESPACE\"}) by (collection, type)", "format": "time_series", "hide": false, "intervalFactor": 2, @@ -2109,7 +2109,7 @@ "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "expr": "sum(SeaweedFS_volumeServer_volumes)", + "expr": "sum(SeaweedFS_volumeServer_volumes{namespace=\"$NAMESPACE\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "Total", @@ -2194,7 +2194,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(SeaweedFS_volumeServer_total_disk_size) by (collection, type)", + "expr": "sum(SeaweedFS_volumeServer_total_disk_size{namespace=\"$NAMESPACE\"}) by (collection, type)", "format": "time_series", "hide": false, "intervalFactor": 2, @@ -2202,7 +2202,7 @@ "refId": "A" }, { - "expr": "sum(SeaweedFS_volumeServer_total_disk_size)", + "expr": "sum(SeaweedFS_volumeServer_total_disk_size{namespace=\"$NAMESPACE\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "Total", @@ -2285,7 +2285,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(SeaweedFS_volumeServer_total_disk_size) by (exported_instance)", + "expr": "sum(SeaweedFS_volumeServer_total_disk_size{namespace=\"$NAMESPACE\"}) by (exported_instance)", "format": "time_series", "hide": false, "intervalFactor": 2, @@ -2390,7 +2390,7 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(SeaweedFS_filerStore_request_seconds_bucket[$__rate_interval])) by (le, type))", + "expr": "histogram_quantile(0.99, sum(rate(SeaweedFS_filerStore_request_seconds_bucket{namespace=\"$NAMESPACE\"}[$__rate_interval])) by (le, type))", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{type}}", @@ -2482,7 +2482,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(SeaweedFS_filerStore_request_total [$__rate_interval])) by (type)", + "expr": "sum(rate(SeaweedFS_filerStore_request_total{namespace=\"$NAMESPACE\"}[$__rate_interval])) by (type)", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{type}}", @@ -2589,7 +2589,7 @@ "targets": [ { "exemplar": true, - "expr": "go_memstats_alloc_bytes{job=\"seaweedfs-filer\"}", + "expr": "go_memstats_alloc_bytes{job=\"seaweedfs-filer\", namespace=\"$NAMESPACE\"}", "format": "time_series", "hide": false, "interval": "", @@ -2599,7 +2599,7 @@ }, { "exemplar": true, - "expr": "rate(go_memstats_alloc_bytes_total{job=\"seaweedfs-filer\"}[$__rate_interval])", + "expr": "rate(go_memstats_alloc_bytes_total{job=\"seaweedfs-filer\", namespace=\"$NAMESPACE\"}[$__rate_interval])", "format": "time_series", "hide": false, "interval": "", @@ -2609,7 +2609,7 @@ }, { "exemplar": true, - "expr": "go_memstats_stack_inuse_bytes{job=\"seaweedfs-filer\"}", + "expr": "go_memstats_stack_inuse_bytes{job=\"seaweedfs-filer\", namespace=\"$NAMESPACE\"}", "format": "time_series", "hide": false, "interval": "", @@ -2619,7 +2619,7 @@ }, { "exemplar": true, - "expr": "go_memstats_heap_inuse_bytes{job=\"seaweedfs-filer\"}", + "expr": "go_memstats_heap_inuse_bytes{job=\"seaweedfs-filer\", namespace=\"$NAMESPACE\"}", "format": "time_series", "hide": false, "interval": "", @@ -2711,7 +2711,7 @@ "targets": [ { "exemplar": true, - "expr": "go_gc_duration_seconds{job=\"seaweedfs-filer\"}", + "expr": "go_gc_duration_seconds{job=\"seaweedfs-filer\", namespace=\"$NAMESPACE\"}", "format": "time_series", "interval": "", "intervalFactor": 2, @@ -2802,7 +2802,7 @@ "targets": [ { "exemplar": true, - "expr": "go_goroutines{job=\"seaweedfs-filer\"}", + "expr": "go_goroutines{job=\"seaweedfs-filer\", namespace=\"$NAMESPACE\"}", "format": "time_series", "interval": "", "intervalFactor": 2, @@ -2867,6 +2867,34 @@ "regex": "", "skipUrlSync": false, "type": "datasource" + }, + { + "current": { + "selected": false, + "text": "mes", + "value": "mes" + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "definition": "label_values(SeaweedFS_master_is_leader,namespace)", + "hide": 0, + "includeAll": false, + "label": "Namespace", + "multi": false, + "name": "NAMESPACE", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(SeaweedFS_master_is_leader,namespace)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 2, + "type": "query" } ] }, From 262fe2c7239253a8420dc4e63860d2adb47f3b9a Mon Sep 17 00:00:00 2001 From: Gregor Tudan Date: Tue, 14 May 2024 14:29:31 +0200 Subject: [PATCH 11/24] Helm: Allow adding additional labels to the service monitors (#5593) --- k8s/charts/seaweedfs/templates/filer-servicemonitor.yaml | 3 +++ k8s/charts/seaweedfs/templates/master-servicemonitor.yaml | 3 +++ k8s/charts/seaweedfs/templates/s3-servicemonitor.yaml | 3 +++ k8s/charts/seaweedfs/templates/volume-servicemonitor.yaml | 3 +++ k8s/charts/seaweedfs/values.yaml | 1 + 5 files changed, 13 insertions(+) diff --git a/k8s/charts/seaweedfs/templates/filer-servicemonitor.yaml b/k8s/charts/seaweedfs/templates/filer-servicemonitor.yaml index d0aa6c8b8..76c981c1a 100644 --- a/k8s/charts/seaweedfs/templates/filer-servicemonitor.yaml +++ b/k8s/charts/seaweedfs/templates/filer-servicemonitor.yaml @@ -12,6 +12,9 @@ metadata: app.kubernetes.io/managed-by: {{ .Release.Service }} app.kubernetes.io/instance: {{ .Release.Name }} app.kubernetes.io/component: filer + {{- with .Values.global.monitoring.additionalLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} spec: endpoints: - interval: 30s diff --git a/k8s/charts/seaweedfs/templates/master-servicemonitor.yaml b/k8s/charts/seaweedfs/templates/master-servicemonitor.yaml index 93715f031..81cade2ef 100644 --- a/k8s/charts/seaweedfs/templates/master-servicemonitor.yaml +++ b/k8s/charts/seaweedfs/templates/master-servicemonitor.yaml @@ -12,6 +12,9 @@ metadata: app.kubernetes.io/managed-by: {{ .Release.Service }} app.kubernetes.io/instance: {{ .Release.Name }} app.kubernetes.io/component: master + {{- with .Values.global.monitoring.additionalLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} spec: endpoints: - interval: 30s diff --git a/k8s/charts/seaweedfs/templates/s3-servicemonitor.yaml b/k8s/charts/seaweedfs/templates/s3-servicemonitor.yaml index 7caf1a167..b47ba8ee6 100644 --- a/k8s/charts/seaweedfs/templates/s3-servicemonitor.yaml +++ b/k8s/charts/seaweedfs/templates/s3-servicemonitor.yaml @@ -12,6 +12,9 @@ metadata: app.kubernetes.io/managed-by: {{ .Release.Service }} app.kubernetes.io/instance: {{ .Release.Name }} app.kubernetes.io/component: s3 + {{- with .Values.global.monitoring.additionalLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} spec: endpoints: - interval: 30s diff --git a/k8s/charts/seaweedfs/templates/volume-servicemonitor.yaml b/k8s/charts/seaweedfs/templates/volume-servicemonitor.yaml index c23e4dec0..4aeacc416 100644 --- a/k8s/charts/seaweedfs/templates/volume-servicemonitor.yaml +++ b/k8s/charts/seaweedfs/templates/volume-servicemonitor.yaml @@ -12,6 +12,9 @@ metadata: app.kubernetes.io/managed-by: {{ .Release.Service }} app.kubernetes.io/instance: {{ .Release.Name }} app.kubernetes.io/component: volume + {{- with .Values.global.monitoring.additionalLabels }} + {{- toYaml . | nindent 4 }} + {{- end }} spec: endpoints: - interval: 30s diff --git a/k8s/charts/seaweedfs/values.yaml b/k8s/charts/seaweedfs/values.yaml index c382371ca..8b8c0d33d 100644 --- a/k8s/charts/seaweedfs/values.yaml +++ b/k8s/charts/seaweedfs/values.yaml @@ -25,6 +25,7 @@ global: enabled: false gatewayHost: null gatewayPort: null + additionalLabels: {} # if enabled will use global.replicationPlacment and override master & filer defaultReplicaPlacement config enableReplication: false # replication type is XYZ: From 1db3bb9a9c9aa5506e27344643cb7e5475018f1e Mon Sep 17 00:00:00 2001 From: Gregor Tudan Date: Tue, 14 May 2024 14:30:52 +0200 Subject: [PATCH 12/24] Helm: Fix warning when rendering with secretExtraEnvVars set (#5594) --- k8s/charts/seaweedfs/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k8s/charts/seaweedfs/values.yaml b/k8s/charts/seaweedfs/values.yaml index 8b8c0d33d..576b7da22 100644 --- a/k8s/charts/seaweedfs/values.yaml +++ b/k8s/charts/seaweedfs/values.yaml @@ -606,7 +606,7 @@ filer: timeoutSeconds: 10 # secret env variables - secretExtraEnvironmentVars: [] + secretExtraEnvironmentVars: {} # WEED_POSTGRES_USERNAME: # secretKeyRef: # name: postgres-credentials From 31653e4b1e78652b7696990993f38abda72af525 Mon Sep 17 00:00:00 2001 From: Gregor Tudan Date: Tue, 14 May 2024 14:33:29 +0200 Subject: [PATCH 13/24] Update the Grafana panels using the to the new time series (#5595) --- .../seaweedfs-grafana-dashboard.json | 3144 +++++++++-------- 1 file changed, 1740 insertions(+), 1404 deletions(-) diff --git a/k8s/charts/seaweedfs/dashboards/seaweedfs-grafana-dashboard.json b/k8s/charts/seaweedfs/dashboards/seaweedfs-grafana-dashboard.json index 47f5212b3..f4e3b0209 100644 --- a/k8s/charts/seaweedfs/dashboards/seaweedfs-grafana-dashboard.json +++ b/k8s/charts/seaweedfs/dashboards/seaweedfs-grafana-dashboard.json @@ -21,8 +21,7 @@ "fiscalYearStartMonth": 0, "gnetId": 10423, "graphTooltip": 0, - "id": 1360, - "iteration": 1693482389821, + "id": 160, "links": [], "liveNow": false, "panels": [ @@ -64,7 +63,8 @@ } ] }, - "unit": "bool_yes_no" + "unit": "bool_yes_no", + "unitScale": true }, "overrides": [] }, @@ -88,10 +88,12 @@ "fields": "", "values": false }, + "showPercentChange": false, "text": {}, - "textMode": "auto" + "textMode": "auto", + "wideLayout": true }, - "pluginVersion": "8.5.15", + "pluginVersion": "10.3.1", "targets": [ { "exemplar": true, @@ -120,6 +122,9 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, @@ -131,6 +136,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 4, @@ -157,7 +163,8 @@ } ] }, - "unit": "none" + "unit": "none", + "unitScale": true }, "overrides": [] }, @@ -173,7 +180,8 @@ "legend": { "calcs": [], "displayMode": "list", - "placement": "bottom" + "placement": "bottom", + "showLegend": true }, "tooltip": { "mode": "single", @@ -210,6 +218,9 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, @@ -221,6 +232,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 4, @@ -247,7 +259,8 @@ } ] }, - "unit": "none" + "unit": "none", + "unitScale": true }, "overrides": [] }, @@ -263,7 +276,8 @@ "legend": { "calcs": [], "displayMode": "list", - "placement": "bottom" + "placement": "bottom", + "showLegend": true }, "tooltip": { "mode": "single", @@ -335,6 +349,9 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, @@ -346,6 +363,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 4, @@ -372,7 +390,8 @@ } ] }, - "unit": "none" + "unit": "none", + "unitScale": true }, "overrides": [] }, @@ -388,7 +407,8 @@ "legend": { "calcs": [], "displayMode": "list", - "placement": "bottom" + "placement": "bottom", + "showLegend": true }, "tooltip": { "mode": "single", @@ -475,6 +495,9 @@ "mode": "palette-classic" }, "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", "axisLabel": "", "axisPlacement": "auto", "barAlignment": 0, @@ -486,6 +509,7 @@ "tooltip": false, "viz": false }, + "insertNulls": false, "lineInterpolation": "linear", "lineWidth": 1, "pointSize": 5, @@ -512,7 +536,8 @@ } ] }, - "unit": "none" + "unit": "none", + "unitScale": true }, "overrides": [] }, @@ -528,7 +553,8 @@ "legend": { "calcs": [], "displayMode": "list", - "placement": "bottom" + "placement": "bottom", + "showLegend": true }, "tooltip": { "mode": "single", @@ -587,7 +613,8 @@ } ] }, - "unit": "bool_yes_no" + "unit": "bool_yes_no", + "unitScale": true }, "overrides": [] }, @@ -611,10 +638,12 @@ "fields": "", "values": false }, + "showPercentChange": false, "text": {}, - "textMode": "auto" + "textMode": "auto", + "wideLayout": true }, - "pluginVersion": "8.5.15", + "pluginVersion": "10.3.1", "targets": [ { "datasource": { @@ -655,52 +684,88 @@ "type": "row" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "editable": true, - "error": false, - "fill": 1, - "fillGradient": 0, - "grid": {}, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s", + "unitScale": true + }, + "overrides": [] + }, "gridPos": { "h": 7, "w": 8, "x": 0, "y": 14 }, - "hiddenSeries": false, "id": 46, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [], - "nullPointMode": "null as zero", "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.5.15", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.3.1", "targets": [ { "expr": "histogram_quantile(0.90, sum(rate(SeaweedFS_filer_request_seconds_bucket{namespace=\"$NAMESPACE\"}[$__rate_interval])) by (le))", @@ -721,85 +786,92 @@ "step": 60 } ], - "thresholds": [], - "timeRegions": [], "title": "Filer Request Duration 90th percentile", - "tooltip": { - "msResolution": true, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "logBase": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "editable": true, - "error": false, - "fill": 1, - "fillGradient": 0, - "grid": {}, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s", + "unitScale": true + }, + "overrides": [] + }, "gridPos": { "h": 7, "w": 8, "x": 8, "y": 14 }, - "hiddenSeries": false, "id": 49, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [], - "nullPointMode": "null as zero", "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.5.15", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.3.1", "targets": [ { "expr": "histogram_quantile(0.95, sum(rate(SeaweedFS_filer_request_seconds_bucket{namespace=\"$NAMESPACE\"}[$__rate_interval])) by (le))", @@ -826,85 +898,92 @@ "refId": "C" } ], - "thresholds": [], - "timeRegions": [], "title": "Filer Request Duration 95th percentile", - "tooltip": { - "msResolution": true, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "logBase": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "editable": true, - "error": false, - "fill": 1, - "fillGradient": 0, - "grid": {}, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s", + "unitScale": true + }, + "overrides": [] + }, "gridPos": { "h": 7, "w": 8, "x": 16, "y": 14 }, - "hiddenSeries": false, "id": 45, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [], - "nullPointMode": "null as zero", "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.5.15", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.3.1", "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(SeaweedFS_filer_request_seconds_bucket{namespace=\"$NAMESPACE\"}[$__rate_interval])) by (le))", @@ -931,99 +1010,149 @@ "refId": "C" } ], - "thresholds": [], - "timeRegions": [], "title": "Filer Request Duration 99th percentile", - "tooltip": { - "msResolution": true, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "logBase": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "editable": true, - "error": false, - "fill": 0, - "fillGradient": 0, - "grid": {}, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short", + "unitScale": true + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "total" + }, + "properties": [ + { + "id": "custom.lineWidth", + "value": 0 + } + ] + }, + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsZero", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + }, + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsNull", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + } + ] + }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 21 }, - "hiddenSeries": false, "id": 2, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": 250, - "sort": "max", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, "links": [], - "maxPerRow": 1, - "nullPointMode": "null as zero", "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.5.15", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "$$hashKey": "object:810", - "alias": "total", - "lines": false + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true, + "width": 250 + }, + "tooltip": { + "mode": "multi", + "sort": "desc" } - ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, + }, + "pluginVersion": "10.3.1", "targets": [ { "exemplar": true, @@ -1036,39 +1165,8 @@ "step": 30 } ], - "thresholds": [], - "timeRegions": [], "title": "Filer QPS", - "tooltip": { - "msResolution": true, - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:817", - "format": "short", - "logBase": 1, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:818", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } + "type": "timeseries" }, { "collapsed": false, @@ -1088,52 +1186,88 @@ "type": "row" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "editable": true, - "error": false, - "fill": 1, - "fillGradient": 0, - "grid": {}, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s", + "unitScale": true + }, + "overrides": [] + }, "gridPos": { "h": 7, "w": 8, "x": 0, "y": 29 }, - "hiddenSeries": false, "id": 65, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [], - "nullPointMode": "null as zero", "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.5.15", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.3.1", "targets": [ { "expr": "histogram_quantile(0.90, sum(rate(SeaweedFS_s3_request_seconds_bucket{namespace=\"$NAMESPACE\"}[$__rate_interval])) by (le))", @@ -1154,85 +1288,92 @@ "step": 60 } ], - "thresholds": [], - "timeRegions": [], "title": "S3 Request Duration 90th percentile", - "tooltip": { - "msResolution": true, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "logBase": 1, + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], "min": 0, - "show": true + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s", + "unitScale": true }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "overrides": [] }, - "editable": true, - "error": false, - "fill": 1, - "fillGradient": 0, - "grid": {}, "gridPos": { "h": 7, "w": 8, "x": 8, "y": 29 }, - "hiddenSeries": false, "id": 56, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [], - "nullPointMode": "null as zero", "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.5.15", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.3.1", "targets": [ { "expr": "histogram_quantile(0.95, sum(rate(SeaweedFS_s3_request_seconds_bucket{namespace=\"$NAMESPACE\"}[$__rate_interval])) by (le))", @@ -1253,85 +1394,92 @@ "step": 60 } ], - "thresholds": [], - "timeRegions": [], "title": "S3 Request Duration 95th percentile", - "tooltip": { - "msResolution": true, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "logBase": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "editable": true, - "error": false, - "fill": 1, - "fillGradient": 0, - "grid": {}, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s", + "unitScale": true + }, + "overrides": [] + }, "gridPos": { "h": 7, "w": 8, "x": 16, "y": 29 }, - "hiddenSeries": false, "id": 58, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [], - "nullPointMode": "null as zero", "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.5.15", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.3.1", "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(SeaweedFS_s3_request_seconds_bucket{namespace=\"$NAMESPACE\"}[$__rate_interval])) by (le))", @@ -1354,87 +1502,92 @@ "step": 60 } ], - "thresholds": [], - "timeRegions": [], "title": "S3 Request Duration 99th percentile", - "tooltip": { - "msResolution": true, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:115", - "format": "s", - "logBase": 1, - "min": 0, - "show": true - }, - { - "$$hashKey": "object:116", - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "editable": true, - "error": false, - "fill": 1, - "fillGradient": 0, - "grid": {}, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s", + "unitScale": true + }, + "overrides": [] + }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 36 }, - "hiddenSeries": false, "id": 72, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [], - "nullPointMode": "null as zero", "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.5.15", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.3.1", "targets": [ { "datasource": { @@ -1467,87 +1620,92 @@ "step": 60 } ], - "thresholds": [], - "timeRegions": [], "title": "S3 Request. Duration 99th percentile per instance", - "tooltip": { - "msResolution": true, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:115", - "format": "s", - "logBase": 1, - "min": 0, - "show": true - }, - { - "$$hashKey": "object:116", - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "editable": true, - "error": false, - "fill": 1, - "fillGradient": 0, - "grid": {}, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s", + "unitScale": true + }, + "overrides": [] + }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 43 }, - "hiddenSeries": false, "id": 73, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [], - "nullPointMode": "null as zero", "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.5.15", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.3.1", "targets": [ { "datasource": { @@ -1565,101 +1723,149 @@ "step": 60 } ], - "thresholds": [], - "timeRegions": [], "title": "S3 Request Duration 99th percentile per bucket", - "tooltip": { - "msResolution": true, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:115", - "format": "s", - "logBase": 1, + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], "min": 0, - "show": true + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short", + "unitScale": true }, - { - "$$hashKey": "object:116", - "format": "short", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": { - "type": "prometheus", - "uid": "${DS_PROMETHEUS}" + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "total" + }, + "properties": [ + { + "id": "custom.lineWidth", + "value": 0 + } + ] + }, + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsZero", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + }, + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsNull", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + } + ] }, - "editable": true, - "error": false, - "fill": 0, - "fillGradient": 0, - "grid": {}, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 50 }, - "hiddenSeries": false, "id": 55, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": 250, - "sort": "max", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, "links": [], - "maxPerRow": 1, - "nullPointMode": "null as zero", "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.5.15", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "$$hashKey": "object:689", - "alias": "total", - "lines": false + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true, + "width": 250 + }, + "tooltip": { + "mode": "multi", + "sort": "desc" } - ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, + }, + "pluginVersion": "10.3.1", "targets": [ { "datasource": { @@ -1674,101 +1880,150 @@ "step": 30 } ], - "thresholds": [], - "timeRegions": [], "title": "S3 API QPS", - "tooltip": { - "msResolution": true, - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:696", - "format": "short", - "logBase": 1, - "min": "0", - "show": true - }, - { - "$$hashKey": "object:697", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "editable": true, - "error": false, - "fill": 0, - "fillGradient": 0, - "grid": {}, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Cost in US$", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "currencyUSD", + "unitScale": true + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "total" + }, + "properties": [ + { + "id": "custom.lineWidth", + "value": 0 + } + ] + }, + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsZero", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + }, + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsNull", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + } + ] + }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 57 }, - "hiddenSeries": false, "hideTimeOverride": false, "id": 59, - "legend": { - "alignAsTable": true, - "avg": false, - "current": true, - "hideEmpty": true, - "hideZero": true, - "max": true, - "min": false, - "rightSide": true, - "show": true, - "sideWidth": 250, - "sort": "max", - "sortDesc": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 1, "links": [], - "maxPerRow": 1, - "nullPointMode": "null as zero", "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.5.15", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "alias": "total", - "lines": false + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true, + "width": 250 + }, + "tooltip": { + "mode": "multi", + "sort": "desc" } - ], - "spaceLength": 10, - "stack": false, - "steppedLine": false, + }, + "pluginVersion": "10.3.1", "targets": [ { "expr": "sum by (type) (SeaweedFS_s3_request_total{type=~'PUT|COPY|POST|LIST', namespace=\"$NAMESPACE\"})*0.000005", @@ -1805,40 +2060,9 @@ "refId": "D" } ], - "thresholds": [], "timeFrom": "1M", - "timeRegions": [], "title": "S3 API Monthly Cost if on AWS", - "tooltip": { - "msResolution": true, - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "currencyUSD", - "label": "Cost in US$", - "logBase": 1, - "min": "0", - "show": true - }, - { - "format": "currencyUSD", - "label": "Write Cost", - "logBase": 1, - "show": false - } - ], - "yaxis": { - "align": false - } + "type": "timeseries" }, { "collapsed": false, @@ -1858,52 +2082,87 @@ "type": "row" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { "uid": "${DS_PROMETHEUS}" }, - "editable": true, - "error": false, - "fill": 1, - "fillGradient": 0, - "grid": {}, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s", + "unitScale": true + }, + "overrides": [] + }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 65 }, - "hiddenSeries": false, "id": 47, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, "links": [], - "nullPointMode": "null as zero", "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.5.15", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.3.1", "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(SeaweedFS_volumeServer_request_seconds_bucket{namespace=\"$NAMESPACE\"}[$__rate_interval])) by (le, exported_instance))", @@ -1921,91 +2180,134 @@ "refId": "C" } ], - "thresholds": [], - "timeRegions": [], "title": "Volume Server Request Duration 99th percentile", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "logBase": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "editable": true, - "error": false, - "fill": 1, - "fillGradient": 0, - "grid": {}, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short", + "unitScale": true + }, + "overrides": [ + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsZero", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + }, + { + "matcher": { + "id": "byValue", + "options": { + "op": "gte", + "reducer": "allIsNull", + "value": 0 + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": true, + "tooltip": true, + "viz": false + } + } + ] + } + ] + }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 65 }, - "hiddenSeries": false, "id": 40, - "legend": { - "alignAsTable": true, - "avg": false, - "current": false, - "hideEmpty": true, - "hideZero": true, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "sort": "total", - "sortDesc": true, - "total": true, - "values": true - }, - "lines": true, - "linewidth": 2, "links": [], - "nullPointMode": "null as zero", "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.5.15", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, + "legend": { + "calcs": [ + "sum" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.3.1", "targets": [ { "expr": "sum(rate(SeaweedFS_volumeServer_request_total{namespace=\"$NAMESPACE\"}[$__rate_interval])) by (type)", @@ -2016,81 +2318,91 @@ "step": 4 } ], - "thresholds": [], - "timeRegions": [], "title": "Volume Server QPS", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "fill": 1, - "fillGradient": 0, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short", + "unitScale": true + }, + "overrides": [] + }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 72 }, - "hiddenSeries": false, "id": 48, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [], - "nullPointMode": "null", "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.5.15", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.3.1", "targets": [ { "datasource": { @@ -2116,82 +2428,91 @@ "refId": "B" } ], - "thresholds": [], - "timeRegions": [], "title": "Volume Count", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "$$hashKey": "object:101", - "format": "short", - "logBase": 1, - "show": true - }, - { - "$$hashKey": "object:102", - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "fill": 1, - "fillGradient": 0, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes", + "unitScale": true + }, + "overrides": [] + }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 79 }, - "hiddenSeries": false, "id": 50, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [], - "nullPointMode": "null", "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.5.15", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.3.1", "targets": [ { "expr": "sum(SeaweedFS_volumeServer_total_disk_size{namespace=\"$NAMESPACE\"}) by (collection, type)", @@ -2209,80 +2530,91 @@ "refId": "B" } ], - "thresholds": [], - "timeRegions": [], "title": "Used Disk Space by Collection and Type", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "fill": 1, - "fillGradient": 0, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes", + "unitScale": true + }, + "overrides": [] + }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 86 }, - "hiddenSeries": false, "id": 51, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [], - "nullPointMode": "null", "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.5.15", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.3.1", "targets": [ { "expr": "sum(SeaweedFS_volumeServer_total_disk_size{namespace=\"$NAMESPACE\"}) by (exported_instance)", @@ -2293,35 +2625,8 @@ "refId": "A" } ], - "thresholds": [], - "timeRegions": [], "title": "Used Disk Space by Host", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "logBase": 1, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } + "type": "timeseries" }, { "collapsed": false, @@ -2341,53 +2646,88 @@ "type": "row" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "editable": true, - "error": false, - "fill": 1, - "fillGradient": 0, - "grid": {}, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s", + "unitScale": true + }, + "overrides": [] + }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 94 }, - "hiddenSeries": false, - "id": 12, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, - "links": [], - "nullPointMode": "null as zero", - "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.5.15", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, + "id": 12, + "links": [], + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.3.1", "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(SeaweedFS_filerStore_request_seconds_bucket{namespace=\"$NAMESPACE\"}[$__rate_interval])) by (le, type))", @@ -2397,89 +2737,95 @@ "refId": "B" } ], - "thresholds": [], - "timeRegions": [], "title": "Filer Store Request Duration 99th percentile", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "logBase": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "editable": true, - "error": false, - "fill": 1, - "fillGradient": 0, - "grid": {}, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short", + "unitScale": true + }, + "overrides": [] + }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 94 }, - "hiddenSeries": false, "id": 14, - "legend": { - "alignAsTable": true, - "avg": true, - "current": true, - "hideEmpty": false, - "hideZero": false, - "max": false, - "min": false, - "rightSide": true, - "show": true, - "total": false, - "values": true - }, - "lines": true, - "linewidth": 2, "links": [], - "nullPointMode": "null as zero", "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.5.15", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, + "legend": { + "calcs": [ + "mean", + "lastNotNull" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.3.1", "targets": [ { "expr": "sum(rate(SeaweedFS_filerStore_request_total{namespace=\"$NAMESPACE\"}[$__rate_interval])) by (type)", @@ -2489,37 +2835,8 @@ "refId": "B" } ], - "thresholds": [], - "timeRegions": [], "title": "Filer Store QPS", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "logBase": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } + "type": "timeseries" }, { "collapsed": false, @@ -2539,53 +2856,88 @@ "type": "row" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "editable": true, - "error": false, - "fill": 1, - "fillGradient": 0, - "grid": {}, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes", + "unitScale": true + }, + "overrides": [] + }, "gridPos": { "h": 7, "w": 12, "x": 0, "y": 102 }, - "hiddenSeries": false, "id": 52, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, "links": [], - "nullPointMode": "null as zero", "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.5.15", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.3.1", "targets": [ { "exemplar": true, @@ -2628,86 +2980,92 @@ "refId": "D" } ], - "thresholds": [], - "timeRegions": [], "title": "Filer Go Memory Stats", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "logBase": 1, - "min": 0, - "show": true - }, - { - "format": "Bps", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "editable": true, - "error": false, - "fill": 1, - "fillGradient": 0, - "grid": {}, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s", + "unitScale": true + }, + "overrides": [] + }, "gridPos": { "h": 7, "w": 12, "x": 12, "y": 102 }, - "hiddenSeries": false, "id": 54, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, "links": [], - "nullPointMode": "null as zero", "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.5.15", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.3.1", "targets": [ { "exemplar": true, @@ -2719,86 +3077,92 @@ "refId": "B" } ], - "thresholds": [], - "timeRegions": [], "title": "Filer Go GC duration quantiles", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "logBase": 1, - "min": 0, - "show": true - }, - { - "format": "Bps", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": { "type": "prometheus", "uid": "${DS_PROMETHEUS}" }, - "editable": true, - "error": false, - "fill": 1, - "fillGradient": 0, - "grid": {}, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 2, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none", + "unitScale": true + }, + "overrides": [] + }, "gridPos": { "h": 7, "w": 24, "x": 0, "y": 109 }, - "hiddenSeries": false, "id": 53, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "max": false, - "min": false, - "show": false, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 2, "links": [], - "nullPointMode": "null as zero", "options": { - "alertThreshold": true - }, - "percentage": false, - "pluginVersion": "8.5.15", - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.3.1", "targets": [ { "exemplar": true, @@ -2810,42 +3174,12 @@ "refId": "B" } ], - "thresholds": [], - "timeRegions": [], "title": "Filer Go Routines", - "tooltip": { - "msResolution": false, - "shared": true, - "sort": 0, - "value_type": "cumulative" - }, - "type": "graph", - "xaxis": { - "mode": "time", - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "none", - "logBase": 1, - "min": 0, - "show": true - }, - { - "format": "short", - "logBase": 1, - "show": true - } - ], - "yaxis": { - "align": false - } + "type": "timeseries" } ], - "refresh": false, - "schemaVersion": 36, - "style": "dark", + "refresh": "", + "schemaVersion": 39, "tags": [], "templating": { "list": [ @@ -2853,7 +3187,7 @@ "current": { "selected": false, "text": "Prometheus", - "value": "Prometheus" + "value": "PBFA97CFB590B2093" }, "hide": 0, "includeAll": false, @@ -2929,5 +3263,7 @@ }, "timezone": "browser", "title": "SeaweedFS", - "version": 3 + "uid": "a24009d7-cbda-4443-a132-1cc1c4677304", + "version": 1, + "weekStart": "" } \ No newline at end of file From 54f3913bedf3145a899f2ef872735dd54e468fcc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Stiborsk=C3=BD?= Date: Tue, 14 May 2024 15:48:24 +0200 Subject: [PATCH 14/24] [s3] Fixed s3 replication by sending content-md as base64 (#5596) --- weed/replication/sink/s3sink/s3_sink.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/weed/replication/sink/s3sink/s3_sink.go b/weed/replication/sink/s3sink/s3_sink.go index 81acd9a2d..6e7549cdc 100644 --- a/weed/replication/sink/s3sink/s3_sink.go +++ b/weed/replication/sink/s3sink/s3_sink.go @@ -11,6 +11,7 @@ import ( "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" "strconv" "strings" + "encoding/base64" "github.com/seaweedfs/seaweedfs/weed/filer" "github.com/seaweedfs/seaweedfs/weed/glog" @@ -202,7 +203,7 @@ func (s3sink *S3Sink) CreateEntry(key string, entry *filer_pb.Entry, signatures Tagging: aws.String(tags), } if len(entry.Attributes.Md5) > 0 { - uploadInput.ContentMD5 = aws.String(fmt.Sprintf("%x", entry.Attributes.Md5)) + uploadInput.ContentMD5 = aws.String(base64.StdEncoding.EncodeToString([]byte(entry.Attributes.Md5))) } _, err = uploader.Upload(&uploadInput) From f6e8a9bf9ce3072cfd1d27157cc2a83f91d669df Mon Sep 17 00:00:00 2001 From: Riccardo Bertossa <33728857+rikigigi@users.noreply.github.com> Date: Fri, 17 May 2024 13:54:09 +0200 Subject: [PATCH 15/24] added s3 iam DeleteBucket permission management (#5599) --- weed/iamapi/iamapi_management_handlers.go | 5 +++++ weed/s3api/auth_credentials.go | 4 ++++ weed/s3api/auth_credentials_test.go | 15 +++++++++++++-- weed/s3api/s3_constants/s3_actions.go | 15 ++++++++------- weed/s3api/s3_constants/s3_config.go | 2 +- weed/s3api/s3api_bucket_handlers.go | 12 +++++++++--- weed/s3api/s3api_server.go | 2 +- 7 files changed, 41 insertions(+), 14 deletions(-) diff --git a/weed/iamapi/iamapi_management_handlers.go b/weed/iamapi/iamapi_management_handlers.go index d63bc8849..6b0f9bbfc 100644 --- a/weed/iamapi/iamapi_management_handlers.go +++ b/weed/iamapi/iamapi_management_handlers.go @@ -33,6 +33,7 @@ const ( StatementActionReadAcp = "GetBucketAcl" StatementActionList = "List*" StatementActionTagging = "Tagging*" + StatementActionDelete = "DeleteBucket*" ) var ( @@ -58,6 +59,8 @@ func MapToStatementAction(action string) string { return s3_constants.ACTION_LIST case StatementActionTagging: return s3_constants.ACTION_TAGGING + case StatementActionDelete: + return s3_constants.ACTION_DELETE_BUCKET default: return "" } @@ -79,6 +82,8 @@ func MapToIdentitiesAction(action string) string { return StatementActionList case s3_constants.ACTION_TAGGING: return StatementActionTagging + case s3_constants.ACTION_DELETE_BUCKET: + return StatementActionDelete default: return "" } diff --git a/weed/s3api/auth_credentials.go b/weed/s3api/auth_credentials.go index a2b1fd90f..6121aecba 100644 --- a/weed/s3api/auth_credentials.go +++ b/weed/s3api/auth_credentials.go @@ -317,6 +317,7 @@ func (iam *IdentityAccessManagement) Auth(f http.HandlerFunc, action Action) htt } identity, errCode := iam.authRequest(r, action) + glog.V(3).Infof("auth error: %v", errCode) if errCode == s3err.ErrNone { if identity != nil && identity.Name != "" { r.Header.Set(s3_constants.AmzIdentityId, identity.Name) @@ -453,6 +454,7 @@ func (identity *Identity) canDo(action Action, bucket string, objectKey string) } } if bucket == "" { + glog.V(3).Infof("identity %s is not allowed to perform action %s on %s -- bucket is empty", identity.Name, action, bucket+objectKey) return false } target := string(action) + ":" + bucket + objectKey @@ -477,6 +479,8 @@ func (identity *Identity) canDo(action Action, bucket string, objectKey string) } } } + //log error + glog.V(3).Infof("identity %s is not allowed to perform action %s on %s", identity.Name, action, bucket+objectKey) return false } diff --git a/weed/s3api/auth_credentials_test.go b/weed/s3api/auth_credentials_test.go index f9f87fc54..1d9f1a95f 100644 --- a/weed/s3api/auth_credentials_test.go +++ b/weed/s3api/auth_credentials_test.go @@ -1,11 +1,12 @@ package s3api import ( - . "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" - "github.com/stretchr/testify/assert" "reflect" "testing" + . "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" + "github.com/stretchr/testify/assert" + "github.com/seaweedfs/seaweedfs/weed/pb/iam_pb" jsonpb "google.golang.org/protobuf/encoding/protojson" ) @@ -79,6 +80,7 @@ func TestCanDo(t *testing.T) { } // object specific assert.Equal(t, true, ident1.canDo(ACTION_WRITE, "bucket1", "/a/b/c/d.txt")) + assert.Equal(t, false, ident1.canDo(ACTION_DELETE_BUCKET, "bucket1", "")) assert.Equal(t, false, ident1.canDo(ACTION_WRITE, "bucket1", "/a/b/other/some"), "action without *") // bucket specific @@ -141,6 +143,15 @@ func TestCanDo(t *testing.T) { }, } assert.Equal(t, true, ident6.canDo(ACTION_READ, "anything_bucket", "/a/b/c/d.txt")) + + //test deleteBucket operation + ident7 := &Identity{ + Name: "anything", + Actions: []Action{ + "DeleteBucket:bucket1", + }, + } + assert.Equal(t, true, ident7.canDo(ACTION_DELETE_BUCKET, "bucket1", "")) } type LoadS3ApiConfigurationTestCase struct { diff --git a/weed/s3api/s3_constants/s3_actions.go b/weed/s3api/s3_constants/s3_actions.go index 8d770e408..864979784 100644 --- a/weed/s3api/s3_constants/s3_actions.go +++ b/weed/s3api/s3_constants/s3_actions.go @@ -1,13 +1,14 @@ package s3_constants const ( - ACTION_READ = "Read" - ACTION_READ_ACP = "ReadAcp" - ACTION_WRITE = "Write" - ACTION_WRITE_ACP = "WriteAcp" - ACTION_ADMIN = "Admin" - ACTION_TAGGING = "Tagging" - ACTION_LIST = "List" + ACTION_READ = "Read" + ACTION_READ_ACP = "ReadAcp" + ACTION_WRITE = "Write" + ACTION_WRITE_ACP = "WriteAcp" + ACTION_ADMIN = "Admin" + ACTION_TAGGING = "Tagging" + ACTION_LIST = "List" + ACTION_DELETE_BUCKET = "DeleteBucket" SeaweedStorageDestinationHeader = "x-seaweedfs-destination" MultipartUploadsFolder = ".uploads" diff --git a/weed/s3api/s3_constants/s3_config.go b/weed/s3api/s3_constants/s3_config.go index cb44b9484..d2d2c257a 100644 --- a/weed/s3api/s3_constants/s3_config.go +++ b/weed/s3api/s3_constants/s3_config.go @@ -7,7 +7,7 @@ import ( var ( CircuitBreakerConfigDir = "/etc/s3" CircuitBreakerConfigFile = "circuit_breaker.json" - AllowedActions = []string{ACTION_READ, ACTION_READ_ACP, ACTION_WRITE, ACTION_WRITE_ACP, ACTION_LIST, ACTION_TAGGING, ACTION_ADMIN} + AllowedActions = []string{ACTION_READ, ACTION_READ_ACP, ACTION_WRITE, ACTION_WRITE_ACP, ACTION_LIST, ACTION_TAGGING, ACTION_ADMIN, ACTION_DELETE_BUCKET} LimitTypeCount = "Count" LimitTypeBytes = "MB" Separator = ":" diff --git a/weed/s3api/s3api_bucket_handlers.go b/weed/s3api/s3api_bucket_handlers.go index 151bdaca5..12d2c0432 100644 --- a/weed/s3api/s3api_bucket_handlers.go +++ b/weed/s3api/s3api_bucket_handlers.go @@ -6,14 +6,15 @@ import ( "encoding/xml" "errors" "fmt" - "github.com/aws/aws-sdk-go/private/protocol/xml/xmlutil" - "github.com/seaweedfs/seaweedfs/weed/s3api/s3bucket" - "github.com/seaweedfs/seaweedfs/weed/util" "math" "net/http" "strings" "time" + "github.com/aws/aws-sdk-go/private/protocol/xml/xmlutil" + "github.com/seaweedfs/seaweedfs/weed/s3api/s3bucket" + "github.com/seaweedfs/seaweedfs/weed/util" + "github.com/seaweedfs/seaweedfs/weed/filer" "github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants" "github.com/seaweedfs/seaweedfs/weed/storage/needle" @@ -218,6 +219,10 @@ func (s3a *S3ApiServer) checkBucket(r *http.Request, bucket string) s3err.ErrorC return s3err.ErrNoSuchBucket } + //if iam is enabled, the access was already checked before + if s3a.iam.isEnabled() { + return s3err.ErrNone + } if !s3a.hasAccess(r, entry) { return s3err.ErrAccessDenied } @@ -236,6 +241,7 @@ func (s3a *S3ApiServer) hasAccess(r *http.Request, entry *filer_pb.Entry) bool { identityId := r.Header.Get(s3_constants.AmzIdentityId) if id, ok := entry.Extended[s3_constants.AmzIdentityId]; ok { if identityId != string(id) { + glog.V(3).Infof("hasAccess: %s != %s (entry.Extended = %v)", identityId, id, entry.Extended) return false } } diff --git a/weed/s3api/s3api_server.go b/weed/s3api/s3api_server.go index 1477d650f..9422318ce 100644 --- a/weed/s3api/s3api_server.go +++ b/weed/s3api/s3api_server.go @@ -279,7 +279,7 @@ func (s3a *S3ApiServer) registerRouter(router *mux.Router) { bucket.Methods("PUT").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.PutBucketHandler, ACTION_ADMIN)), "PUT")) // DeleteBucket - bucket.Methods("DELETE").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.DeleteBucketHandler, ACTION_ADMIN)), "DELETE")) + bucket.Methods("DELETE").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.DeleteBucketHandler, ACTION_DELETE_BUCKET)), "DELETE")) // ListObjectsV1 (Legacy) bucket.Methods("GET").HandlerFunc(track(s3a.iam.Auth(s3a.cb.Limit(s3a.ListObjectsV1Handler, ACTION_LIST)), "LIST")) From 8c59348d2824cfe7abfd9b1ef5dd509c2d5162b8 Mon Sep 17 00:00:00 2001 From: Johnny Cederholm Date: Fri, 17 May 2024 22:06:00 +0200 Subject: [PATCH 16/24] Add ability to configure security context for the different components (#5600) --- .../templates/filer-statefulset.yaml | 6 ++ .../templates/master-statefulset.yaml | 6 ++ .../templates/post-install-bucket-hook.yaml | 6 ++ .../seaweedfs/templates/s3-deployment.yaml | 6 ++ .../templates/volume-statefulset.yaml | 6 ++ k8s/charts/seaweedfs/values.yaml | 76 +++++++++++++++++++ 6 files changed, 106 insertions(+) diff --git a/k8s/charts/seaweedfs/templates/filer-statefulset.yaml b/k8s/charts/seaweedfs/templates/filer-statefulset.yaml index 30f2da537..d83d3a639 100644 --- a/k8s/charts/seaweedfs/templates/filer-statefulset.yaml +++ b/k8s/charts/seaweedfs/templates/filer-statefulset.yaml @@ -62,6 +62,9 @@ spec: initContainers: {{ tpl .Values.filer.initContainers . | nindent 8 | trim }} {{- end }} + {{- if .Values.filer.podSecurityContext.enabled }} + securityContext: {{- omit .Values.filer.podSecurityContext "enabled" | toYaml | nindent 8 }} + {{- end }} containers: - name: seaweedfs image: {{ template "filer.image" . }} @@ -264,6 +267,9 @@ spec: resources: {{ tpl .Values.filer.resources . | nindent 12 | trim }} {{- end }} + {{- if .Values.filer.containerSecurityContext.enabled }} + securityContext: {{- omit .Values.filer.containerSecurityContext "enabled" | toYaml | nindent 12 }} + {{- end }} {{- if .Values.filer.sidecars }} {{- include "common.tplvalues.render" (dict "value" .Values.filer.sidecars "context" $) | nindent 8 }} {{- end }} diff --git a/k8s/charts/seaweedfs/templates/master-statefulset.yaml b/k8s/charts/seaweedfs/templates/master-statefulset.yaml index 61252c5a3..1f196fb9c 100644 --- a/k8s/charts/seaweedfs/templates/master-statefulset.yaml +++ b/k8s/charts/seaweedfs/templates/master-statefulset.yaml @@ -61,6 +61,9 @@ spec: initContainers: {{ tpl .Values.master.initContainers . | nindent 8 | trim }} {{- end }} + {{- if .Values.master.podSecurityContext.enabled }} + securityContext: {{- omit .Values.master.podSecurityContext "enabled" | toYaml | nindent 8 }} + {{- end }} containers: - name: seaweedfs image: {{ template "master.image" . }} @@ -222,6 +225,9 @@ spec: resources: {{ tpl .Values.master.resources . | nindent 12 | trim }} {{- end }} + {{- if .Values.master.containerSecurityContext.enabled }} + securityContext: {{- omit .Values.master.containerSecurityContext "enabled" | toYaml | nindent 12 }} + {{- end }} {{- if .Values.master.sidecars }} {{- include "common.tplvalues.render" (dict "value" .Values.master.sidecars "context" $) | nindent 8 }} {{- end }} diff --git a/k8s/charts/seaweedfs/templates/post-install-bucket-hook.yaml b/k8s/charts/seaweedfs/templates/post-install-bucket-hook.yaml index d9f7e6bcf..2260bd84a 100644 --- a/k8s/charts/seaweedfs/templates/post-install-bucket-hook.yaml +++ b/k8s/charts/seaweedfs/templates/post-install-bucket-hook.yaml @@ -22,6 +22,9 @@ spec: app.kubernetes.io/instance: {{ .Release.Name | quote }} spec: restartPolicy: Never + {{- if .Values.filer.podSecurityContext.enabled }} + securityContext: {{- omit .Values.filer.podSecurityContext "enabled" | toYaml | nindent 8 }} + {{- end }} containers: - name: post-install-job image: {{ template "master.image" . }} @@ -80,6 +83,9 @@ spec: {{- end }} - containerPort: {{ .Values.master.grpcPort }} #name: swfs-master-grpc + {{- if .Values.filer.containerSecurityContext.enabled }} + securityContext: {{- omit .Values.filer.containerSecurityContext "enabled" | toYaml | nindent 12 }} + {{- end }} {{- if .Values.filer.s3.enableAuth }} volumes: - name: config-users diff --git a/k8s/charts/seaweedfs/templates/s3-deployment.yaml b/k8s/charts/seaweedfs/templates/s3-deployment.yaml index 2882e863e..7a02524a0 100644 --- a/k8s/charts/seaweedfs/templates/s3-deployment.yaml +++ b/k8s/charts/seaweedfs/templates/s3-deployment.yaml @@ -50,6 +50,9 @@ spec: initContainers: {{ tpl .Values.s3.initContainers . | nindent 8 | trim }} {{- end }} + {{- if .Values.s3.podSecurityContext.enabled }} + securityContext: {{- omit .Values.s3.podSecurityContext "enabled" | toYaml | nindent 8 }} + {{- end }} containers: - name: seaweedfs image: {{ template "s3.image" . }} @@ -199,6 +202,9 @@ spec: resources: {{ tpl .Values.s3.resources . | nindent 12 | trim }} {{- end }} + {{- if .Values.s3.containerSecurityContext.enabled }} + securityContext: {{- omit .Values.s3.containerSecurityContext "enabled" | toYaml | nindent 12 }} + {{- end }} {{- if .Values.s3.sidecars }} {{- include "common.tplvalues.render" (dict "value" .Values.s3.sidecars "context" $) | nindent 8 }} {{- end }} diff --git a/k8s/charts/seaweedfs/templates/volume-statefulset.yaml b/k8s/charts/seaweedfs/templates/volume-statefulset.yaml index 688b4cb16..b691f3045 100644 --- a/k8s/charts/seaweedfs/templates/volume-statefulset.yaml +++ b/k8s/charts/seaweedfs/templates/volume-statefulset.yaml @@ -72,6 +72,9 @@ spec: {{ tpl .Values.volume.initContainers . | nindent 8 | trim }} {{- end }} {{- end }} + {{- if .Values.volume.podSecurityContext.enabled }} + securityContext: {{- omit .Values.volume.podSecurityContext "enabled" | toYaml | nindent 8 }} + {{- end }} containers: - name: seaweedfs image: {{ template "volume.image" . }} @@ -237,6 +240,9 @@ spec: resources: {{ tpl .Values.volume.resources . | nindent 12 | trim }} {{- end }} + {{- if .Values.volume.containerSecurityContext.enabled }} + securityContext: {{- omit .Values.volume.containerSecurityContext "enabled" | toYaml | nindent 12 }} + {{- end }} {{- if .Values.volume.sidecars }} {{- include "common.tplvalues.render" (dict "value" .Values.volume.sidecars "context" $) | nindent 8 }} {{- end }} diff --git a/k8s/charts/seaweedfs/values.yaml b/k8s/charts/seaweedfs/values.yaml index 576b7da22..8962e8d34 100644 --- a/k8s/charts/seaweedfs/values.yaml +++ b/k8s/charts/seaweedfs/values.yaml @@ -167,6 +167,25 @@ master: # ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ serviceAccountName: "" + # Configure security context for Pod + # ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ + # Example: + # podSecurityContext: + # enabled: true + # runAsUser: 1000 + # runAsGroup: 3000 + # fsGroup: 2000 + podSecurityContext: {} + + # Configure security context for Container + # ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ + # Example: + # containerSecurityContext: + # enabled: true + # runAsUser: 2000 + # allowPrivilegeEscalation: false + containerSecurityContext: {} + ingress: enabled: false className: "nginx" @@ -378,6 +397,25 @@ volume: extraEnvironmentVars: + # Configure security context for Pod + # ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ + # Example: + # podSecurityContext: + # enabled: true + # runAsUser: 1000 + # runAsGroup: 3000 + # fsGroup: 2000 + podSecurityContext: {} + + # Configure security context for Container + # ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ + # Example: + # containerSecurityContext: + # enabled: true + # runAsUser: 2000 + # allowPrivilegeEscalation: false + containerSecurityContext: {} + # used to configure livenessProbe on volume-server containers # livenessProbe: @@ -536,6 +574,25 @@ filer: # ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ serviceAccountName: "" + # Configure security context for Pod + # ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ + # Example: + # podSecurityContext: + # enabled: true + # runAsUser: 1000 + # runAsGroup: 3000 + # fsGroup: 2000 + podSecurityContext: {} + + # Configure security context for Container + # ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ + # Example: + # containerSecurityContext: + # enabled: true + # runAsUser: 2000 + # allowPrivilegeEscalation: false + containerSecurityContext: {} + ingress: enabled: false className: "nginx" @@ -707,6 +764,25 @@ s3: # ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ serviceAccountName: "" + # Configure security context for Pod + # ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ + # Example: + # podSecurityContext: + # enabled: true + # runAsUser: 1000 + # runAsGroup: 3000 + # fsGroup: 2000 + podSecurityContext: {} + + # Configure security context for Container + # ref: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/ + # Example: + # containerSecurityContext: + # enabled: true + # runAsUser: 2000 + # allowPrivilegeEscalation: false + containerSecurityContext: {} + logs: type: "hostPath" size: "" From 3fae87632f023ee971a869f89075faf4fb94dbe4 Mon Sep 17 00:00:00 2001 From: sb <12952877+baursn@users.noreply.github.com> Date: Sat, 18 May 2024 18:02:03 +0200 Subject: [PATCH 17/24] add filer configuration options for username/password and tls (#5601) --- weed/command/scaffold/filer.toml | 6 +++ weed/filer/mongodb/mongodb_store.go | 71 ++++++++++++++++++++++++----- 2 files changed, 66 insertions(+), 11 deletions(-) diff --git a/weed/command/scaffold/filer.toml b/weed/command/scaffold/filer.toml index 30a9cae51..122165cd4 100644 --- a/weed/command/scaffold/filer.toml +++ b/weed/command/scaffold/filer.toml @@ -280,6 +280,12 @@ tls_client_key_file="" [mongodb] enabled = false uri = "mongodb://localhost:27017" +username = "" +password = "" +ssl = false +ssl_ca_file = "" +ssl_cert_file = "" +ssl_key_file = " option_pool_size = 0 database = "seaweedfs" diff --git a/weed/filer/mongodb/mongodb_store.go b/weed/filer/mongodb/mongodb_store.go index 8ebbc6ab0..a1bed3e2d 100644 --- a/weed/filer/mongodb/mongodb_store.go +++ b/weed/filer/mongodb/mongodb_store.go @@ -2,7 +2,12 @@ package mongodb import ( "context" + "crypto/tls" + "crypto/x509" "fmt" + "os" + "time" + "github.com/seaweedfs/seaweedfs/weed/filer" "github.com/seaweedfs/seaweedfs/weed/glog" "github.com/seaweedfs/seaweedfs/weed/pb/filer_pb" @@ -10,7 +15,6 @@ import ( "go.mongodb.org/mongo-driver/bson" "go.mongodb.org/mongo-driver/mongo" "go.mongodb.org/mongo-driver/mongo/options" - "time" ) func init() { @@ -37,17 +41,43 @@ func (store *MongodbStore) Initialize(configuration util.Configuration, prefix s store.database = configuration.GetString(prefix + "database") store.collectionName = "filemeta" poolSize := configuration.GetInt(prefix + "option_pool_size") - return store.connection(configuration.GetString(prefix+"uri"), uint64(poolSize)) + uri := configuration.GetString(prefix + "uri") + ssl := configuration.GetBool(prefix + "ssl") + sslCAFile := configuration.GetString(prefix + "ssl_ca_file") + sslCertFile := configuration.GetString(prefix + "ssl_cert_file") + sslKeyFile := configuration.GetString(prefix + "ssl_key_file") + username := configuration.GetString(prefix + "username") + password := configuration.GetString(prefix + "password") + + return store.connection(uri, uint64(poolSize), ssl, sslCAFile, sslCertFile, sslKeyFile, username, password) } -func (store *MongodbStore) connection(uri string, poolSize uint64) (err error) { - ctx, _ := context.WithTimeout(context.Background(), 10*time.Second) +func (store *MongodbStore) connection(uri string, poolSize uint64, ssl bool, sslCAFile, sslCertFile, sslKeyFile string, username, password string) (err error) { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + opts := options.Client().ApplyURI(uri) if poolSize > 0 { opts.SetMaxPoolSize(poolSize) } + if ssl { + tlsConfig, err := configureTLS(sslCAFile, sslCertFile, sslKeyFile) + if err != nil { + return err + } + opts.SetTLSConfig(tlsConfig) + } + + if username != "" && password != "" { + creds := options.Credential{ + Username: username, + Password: password, + } + opts.SetAuth(creds) + } + client, err := mongo.Connect(ctx, opts) if err != nil { return err @@ -55,10 +85,36 @@ func (store *MongodbStore) connection(uri string, poolSize uint64) (err error) { c := client.Database(store.database).Collection(store.collectionName) err = store.indexUnique(c) + store.connect = client return err } +func configureTLS(caFile, certFile, keyFile string) (*tls.Config, error) { + cert, err := tls.LoadX509KeyPair(certFile, keyFile) + if err != nil { + return nil, fmt.Errorf("could not load client key pair: %s", err) + } + + caCert, err := os.ReadFile(caFile) + if err != nil { + return nil, fmt.Errorf("could not read CA certificate: %s", err) + } + + caCertPool := x509.NewCertPool() + if !caCertPool.AppendCertsFromPEM(caCert) { + return nil, fmt.Errorf("failed to append CA certificate") + } + + tlsConfig := &tls.Config{ + Certificates: []tls.Certificate{cert}, + RootCAs: caCertPool, + InsecureSkipVerify: true, + } + + return tlsConfig, nil +} + func (store *MongodbStore) createIndex(c *mongo.Collection, index mongo.IndexModel, opts *options.CreateIndexesOptions) error { _, err := c.Indexes().CreateOne(context.Background(), index, opts) return err @@ -93,13 +149,10 @@ func (store *MongodbStore) RollbackTransaction(ctx context.Context) error { } func (store *MongodbStore) InsertEntry(ctx context.Context, entry *filer.Entry) (err error) { - return store.UpdateEntry(ctx, entry) - } func (store *MongodbStore) UpdateEntry(ctx context.Context, entry *filer.Entry) (err error) { - dir, name := entry.FullPath.DirAndName() meta, err := entry.EncodeAttributesAndChunks() if err != nil { @@ -126,7 +179,6 @@ func (store *MongodbStore) UpdateEntry(ctx context.Context, entry *filer.Entry) } func (store *MongodbStore) FindEntry(ctx context.Context, fullpath util.FullPath) (entry *filer.Entry, err error) { - dir, name := fullpath.DirAndName() var data Model @@ -154,7 +206,6 @@ func (store *MongodbStore) FindEntry(ctx context.Context, fullpath util.FullPath } func (store *MongodbStore) DeleteEntry(ctx context.Context, fullpath util.FullPath) error { - dir, name := fullpath.DirAndName() where := bson.M{"directory": dir, "name": name} @@ -167,7 +218,6 @@ func (store *MongodbStore) DeleteEntry(ctx context.Context, fullpath util.FullPa } func (store *MongodbStore) DeleteFolderChildren(ctx context.Context, fullpath util.FullPath) error { - where := bson.M{"directory": fullpath} _, err := store.connect.Database(store.database).Collection(store.collectionName).DeleteMany(ctx, where) if err != nil { @@ -182,7 +232,6 @@ func (store *MongodbStore) ListDirectoryPrefixedEntries(ctx context.Context, dir } func (store *MongodbStore) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) { - var where = bson.M{"directory": string(dirPath), "name": bson.M{"$gt": startFileName}} if includeStartFile { where["name"] = bson.M{ From a3a00d94995e25af807a0f7d27531e1f26ae007a Mon Sep 17 00:00:00 2001 From: sb <12952877+baursn@users.noreply.github.com> Date: Sat, 18 May 2024 21:19:10 +0200 Subject: [PATCH 18/24] Feature/mongodb security (#5602) --- weed/command/scaffold/filer.toml | 1 + weed/filer/mongodb/mongodb_store.go | 11 ++++++----- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/weed/command/scaffold/filer.toml b/weed/command/scaffold/filer.toml index 122165cd4..574125207 100644 --- a/weed/command/scaffold/filer.toml +++ b/weed/command/scaffold/filer.toml @@ -286,6 +286,7 @@ ssl = false ssl_ca_file = "" ssl_cert_file = "" ssl_key_file = " +insecure_skip_verify = false option_pool_size = 0 database = "seaweedfs" diff --git a/weed/filer/mongodb/mongodb_store.go b/weed/filer/mongodb/mongodb_store.go index a1bed3e2d..fbaa464b9 100644 --- a/weed/filer/mongodb/mongodb_store.go +++ b/weed/filer/mongodb/mongodb_store.go @@ -48,11 +48,12 @@ func (store *MongodbStore) Initialize(configuration util.Configuration, prefix s sslKeyFile := configuration.GetString(prefix + "ssl_key_file") username := configuration.GetString(prefix + "username") password := configuration.GetString(prefix + "password") + insecure_skip_verify := configuration.GetBool(prefix + "insecure_skip_verify") - return store.connection(uri, uint64(poolSize), ssl, sslCAFile, sslCertFile, sslKeyFile, username, password) + return store.connection(uri, uint64(poolSize), ssl, sslCAFile, sslCertFile, sslKeyFile, username, password, insecure_skip_verify) } -func (store *MongodbStore) connection(uri string, poolSize uint64, ssl bool, sslCAFile, sslCertFile, sslKeyFile string, username, password string) (err error) { +func (store *MongodbStore) connection(uri string, poolSize uint64, ssl bool, sslCAFile, sslCertFile, sslKeyFile string, username, password string, insecure bool) (err error) { ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() @@ -63,7 +64,7 @@ func (store *MongodbStore) connection(uri string, poolSize uint64, ssl bool, ssl } if ssl { - tlsConfig, err := configureTLS(sslCAFile, sslCertFile, sslKeyFile) + tlsConfig, err := configureTLS(sslCAFile, sslCertFile, sslKeyFile, insecure) if err != nil { return err } @@ -90,7 +91,7 @@ func (store *MongodbStore) connection(uri string, poolSize uint64, ssl bool, ssl return err } -func configureTLS(caFile, certFile, keyFile string) (*tls.Config, error) { +func configureTLS(caFile, certFile, keyFile string, insecure bool) (*tls.Config, error) { cert, err := tls.LoadX509KeyPair(certFile, keyFile) if err != nil { return nil, fmt.Errorf("could not load client key pair: %s", err) @@ -109,7 +110,7 @@ func configureTLS(caFile, certFile, keyFile string) (*tls.Config, error) { tlsConfig := &tls.Config{ Certificates: []tls.Certificate{cert}, RootCAs: caCertPool, - InsecureSkipVerify: true, + InsecureSkipVerify: insecure, } return tlsConfig, nil From 07a289785af3c23f1fa2755fc536dbe8d2bbd5db Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 20 May 2024 08:55:56 -0700 Subject: [PATCH 19/24] chore(deps): bump google.golang.org/protobuf from 1.34.0 to 1.34.1 (#5604) Bumps google.golang.org/protobuf from 1.34.0 to 1.34.1. --- updated-dependencies: - dependency-name: google.golang.org/protobuf dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 3 +-- go.sum | 5 ++--- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index b2c2ae7de..f54d3d81b 100644 --- a/go.mod +++ b/go.mod @@ -116,7 +116,7 @@ require ( google.golang.org/api v0.177.0 google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda // indirect google.golang.org/grpc v1.63.2 - google.golang.org/protobuf v1.34.0 + google.golang.org/protobuf v1.34.1 gopkg.in/inf.v0 v0.9.1 // indirect modernc.org/b v1.0.0 // indirect modernc.org/libc v1.49.3 // indirect @@ -276,7 +276,6 @@ require ( github.com/pelletier/go-toml/v2 v2.1.0 // indirect github.com/pengsrc/go-shared v0.2.1-0.20190131101655-1999055a4a14 // indirect github.com/philhofer/fwd v1.1.2 // indirect - github.com/pierrec/lz4/v4 v4.1.18 // indirect github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c // indirect github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c // indirect github.com/pingcap/kvproto v0.0.0-20230403051650-e166ae588106 // indirect diff --git a/go.sum b/go.sum index b3bb772ae..ecbb3aee2 100644 --- a/go.sum +++ b/go.sum @@ -737,7 +737,6 @@ github.com/peterh/liner v1.2.2/go.mod h1:xFwJyiKIXJZUKItq5dGHZSTBRAuG/CpeNpWLyiN github.com/philhofer/fwd v1.1.2 h1:bnDivRJ1EWPjUIRXV5KfORO897HTbpFAQddBdE8t7Gw= github.com/philhofer/fwd v1.1.2/go.mod h1:qkPdfjR2SIEbspLqpe1tO4n5yICnr2DY7mqEx2tUTP0= github.com/pierrec/lz4/v4 v4.1.18 h1:xaKrnTkyoqfh1YItXl56+6KJNVYWlEEPuAQW9xsplYQ= -github.com/pierrec/lz4/v4 v4.1.18/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pingcap/errors v0.11.0/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c h1:xpW9bvK+HuuTmyFqUwr+jcCvpVkK7sumiz+ko5H9eq4= @@ -1475,8 +1474,8 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.34.0 h1:Qo/qEd2RZPCf2nKuorzksSknv0d3ERwp1vFG38gSmH4= -google.golang.org/protobuf v1.34.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg= +google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= From 50ec3d7385f98e7f187cc370cdfae784033dca3d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 20 May 2024 08:56:12 -0700 Subject: [PATCH 20/24] chore(deps): bump google.golang.org/api from 0.177.0 to 0.181.0 (#5605) Bumps [google.golang.org/api](https://github.com/googleapis/google-api-go-client) from 0.177.0 to 0.181.0. - [Release notes](https://github.com/googleapis/google-api-go-client/releases) - [Changelog](https://github.com/googleapis/google-api-go-client/blob/main/CHANGES.md) - [Commits](https://github.com/googleapis/google-api-go-client/compare/v0.177.0...v0.181.0) --- updated-dependencies: - dependency-name: google.golang.org/api dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 14 +++++++------- go.sum | 32 ++++++++++++++++---------------- 2 files changed, 23 insertions(+), 23 deletions(-) diff --git a/go.mod b/go.mod index f54d3d81b..6a71b98b5 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/seaweedfs/seaweedfs go 1.22.0 require ( - cloud.google.com/go v0.112.2 // indirect + cloud.google.com/go v0.113.0 // indirect cloud.google.com/go/pubsub v1.38.0 cloud.google.com/go/storage v1.40.0 github.com/Azure/azure-pipeline-go v0.2.3 @@ -41,7 +41,7 @@ require ( github.com/google/btree v1.1.2 github.com/google/uuid v1.6.0 github.com/google/wire v0.6.0 // indirect - github.com/googleapis/gax-go/v2 v2.12.3 // indirect + github.com/googleapis/gax-go/v2 v2.12.4 // indirect github.com/gorilla/mux v1.8.1 github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed // indirect github.com/hashicorp/errwrap v1.1.0 // indirect @@ -108,12 +108,12 @@ require ( golang.org/x/exp v0.0.0-20240112132812-db7319d0e0e3 golang.org/x/image v0.16.0 golang.org/x/net v0.25.0 - golang.org/x/oauth2 v0.19.0 // indirect + golang.org/x/oauth2 v0.20.0 // indirect golang.org/x/sys v0.20.0 golang.org/x/text v0.15.0 // indirect golang.org/x/tools v0.21.0 golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect - google.golang.org/api v0.177.0 + google.golang.org/api v0.181.0 google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda // indirect google.golang.org/grpc v1.63.2 google.golang.org/protobuf v1.34.1 @@ -159,7 +159,7 @@ require ( ) require ( - cloud.google.com/go/auth v0.3.0 // indirect + cloud.google.com/go/auth v0.4.1 // indirect cloud.google.com/go/auth/oauth2adapt v0.2.2 // indirect cloud.google.com/go/compute/metadata v0.3.0 // indirect cloud.google.com/go/iam v1.1.7 // indirect @@ -328,8 +328,8 @@ require ( golang.org/x/sync v0.7.0 // indirect golang.org/x/term v0.20.0 // indirect golang.org/x/time v0.5.0 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20240429193739-8cf5692501f6 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240429193739-8cf5692501f6 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240506185236-b8a5c65736ae // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240513163218-0867130af1f8 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect gopkg.in/validator.v2 v2.0.1 // indirect diff --git a/go.sum b/go.sum index ecbb3aee2..a0c312d0c 100644 --- a/go.sum +++ b/go.sum @@ -14,10 +14,10 @@ cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZ cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= cloud.google.com/go v0.63.0/go.mod h1:GmezbQc7T2snqkEXWfZ0sy0VfkB/ivI2DdtJL2DEmlg= cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= -cloud.google.com/go v0.112.2 h1:ZaGT6LiG7dBzi6zNOvVZwacaXlmf3lRqnC4DQzqyRQw= -cloud.google.com/go v0.112.2/go.mod h1:iEqjp//KquGIJV/m+Pk3xecgKNhV+ry+vVTsy4TbDms= -cloud.google.com/go/auth v0.3.0 h1:PRyzEpGfx/Z9e8+lHsbkoUVXD0gnu4MNmm7Gp8TQNIs= -cloud.google.com/go/auth v0.3.0/go.mod h1:lBv6NKTWp8E3LPzmO1TbiiRKc4drLOfHsgmlH9ogv5w= +cloud.google.com/go v0.113.0 h1:g3C70mn3lWfckKBiCVsAshabrDg01pQ0pnX1MNtnMkA= +cloud.google.com/go v0.113.0/go.mod h1:glEqlogERKYeePz6ZdkcLJ28Q2I6aERgDDErBg9GzO8= +cloud.google.com/go/auth v0.4.1 h1:Z7YNIhlWRtrnKlZke7z3GMqzvuYzdc2z98F9D1NV5Hg= +cloud.google.com/go/auth v0.4.1/go.mod h1:QVBuVEKpCn4Zp58hzRGvL0tjRGU0YqdRTdCHM1IHnro= cloud.google.com/go/auth/oauth2adapt v0.2.2 h1:+TTV8aXpjeChS9M+aTtN/TjdQnzJvmzKFt//oWu7HX4= cloud.google.com/go/auth/oauth2adapt v0.2.2/go.mod h1:wcYjgpZI9+Yu7LyYBg4pqSiaRkfEK3GQcpb7C/uyF1Q= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= @@ -464,8 +464,8 @@ github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/ github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= -github.com/google/martian/v3 v3.3.2 h1:IqNFLAmvJOgVlpdEBiQbDc2EwKW77amAycfTuWKdfvw= -github.com/google/martian/v3 v3.3.2/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= +github.com/google/martian/v3 v3.3.3 h1:DIhPTQrbPkgs2yJYdXU/eNACCG5DVQjySNRNlflZ9Fc= +github.com/google/martian/v3 v3.3.3/go.mod h1:iEPrYcgCF7jA9OtScMFQyAlZZ4YXTKEtJ1E6RWzmBA0= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= @@ -490,8 +490,8 @@ github.com/googleapis/enterprise-certificate-proxy v0.3.2 h1:Vie5ybvEvT75RniqhfF github.com/googleapis/enterprise-certificate-proxy v0.3.2/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/googleapis/gax-go/v2 v2.12.3 h1:5/zPPDvw8Q1SuXjrqrZslrqT7dL/uJT2CQii/cLCKqA= -github.com/googleapis/gax-go/v2 v2.12.3/go.mod h1:AKloxT6GtNbaLm8QTNSidHUVsHYcBHwWRvkNFJUQcS4= +github.com/googleapis/gax-go/v2 v2.12.4 h1:9gWcmF85Wvq4ryPFvGFaOgPIs1AQX0d0bcbGw4Z96qg= +github.com/googleapis/gax-go/v2 v2.12.4/go.mod h1:KYEYLorsnIGDi/rPC8b5TdlB9kbKoFubselGIoBMCwI= github.com/gopherjs/gopherjs v1.17.2 h1:fQnZVsXk8uxXIStYb0N4bGk7jeyTalG/wsZjQ25dO0g= github.com/gopherjs/gopherjs v1.17.2/go.mod h1:pRRIvn/QzFLrKfvEz3qUuEhtE/zLCWfreZ6J5gM2i+k= github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= @@ -1175,8 +1175,8 @@ golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4Iltr golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.19.0 h1:9+E/EZBCbTLNrbN35fHv/a/d/mOBatymz1zbtQrXpIg= -golang.org/x/oauth2 v0.19.0/go.mod h1:vYi7skDa1x015PmRRYZ7+s1cWyPgrPiSYRe4rnsexc8= +golang.org/x/oauth2 v0.20.0 h1:4mQdhULixXKP1rwYBW0vAijoXnkTG0BLCDRzfe1idMo= +golang.org/x/oauth2 v0.20.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -1384,8 +1384,8 @@ google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0M google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= -google.golang.org/api v0.177.0 h1:8a0p/BbPa65GlqGWtUKxot4p0TV8OGOfyTjtmkXNXmk= -google.golang.org/api v0.177.0/go.mod h1:srbhue4MLjkjbkux5p3dw/ocYOSZTaIEvf7bCOnFQDw= +google.golang.org/api v0.181.0 h1:rPdjwnWgiPPOJx3IcSAQ2III5aX5tCer6wMpa/xmZi4= +google.golang.org/api v0.181.0/go.mod h1:MnQ+M0CFsfUwA5beZ+g/vCBCPXvtmZwRz2qzZk8ih1k= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -1428,10 +1428,10 @@ google.golang.org/genproto v0.0.0-20210903162649-d08c68adba83/go.mod h1:eFjDcFEc google.golang.org/genproto v0.0.0-20220421151946-72621c1f0bd3/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda h1:wu/KJm9KJwpfHWhkkZGohVC6KRrc1oJNr4jwtQMOQXw= google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda/go.mod h1:g2LLCvCeCSir/JJSWosk19BR4NVxGqHUC6rxIRsd7Aw= -google.golang.org/genproto/googleapis/api v0.0.0-20240429193739-8cf5692501f6 h1:DTJM0R8LECCgFeUwApvcEJHz85HLagW8uRENYxHh1ww= -google.golang.org/genproto/googleapis/api v0.0.0-20240429193739-8cf5692501f6/go.mod h1:10yRODfgim2/T8csjQsMPgZOMvtytXKTDRzH6HRGzRw= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240429193739-8cf5692501f6 h1:DujSIu+2tC9Ht0aPNA7jgj23Iq8Ewi5sgkQ++wdvonE= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240429193739-8cf5692501f6/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY= +google.golang.org/genproto/googleapis/api v0.0.0-20240506185236-b8a5c65736ae h1:AH34z6WAGVNkllnKs5raNq3yRq93VnjBG6rpfub/jYk= +google.golang.org/genproto/googleapis/api v0.0.0-20240506185236-b8a5c65736ae/go.mod h1:FfiGhwUm6CJviekPrc0oJ+7h29e+DmWU6UtjX0ZvI7Y= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240513163218-0867130af1f8 h1:mxSlqyb8ZAHsYDCfiXN1EDdNTdvjUJSLY+OnAUtYNYA= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240513163218-0867130af1f8/go.mod h1:I7Y+G38R2bu5j1aLzfFmQfTcU/WnFuqDwLZAbvKTKpM= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= From f483d8b8f4ff47792db5da4c4d1bd7b0304d3bcb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 20 May 2024 08:56:21 -0700 Subject: [PATCH 21/24] chore(deps): bump github.com/aws/aws-sdk-go-v2 from 1.26.1 to 1.27.0 (#5606) Bumps [github.com/aws/aws-sdk-go-v2](https://github.com/aws/aws-sdk-go-v2) from 1.26.1 to 1.27.0. - [Release notes](https://github.com/aws/aws-sdk-go-v2/releases) - [Commits](https://github.com/aws/aws-sdk-go-v2/compare/v1.26.1...v1.27.0) --- updated-dependencies: - dependency-name: github.com/aws/aws-sdk-go-v2 dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 6a71b98b5..1672e8e62 100644 --- a/go.mod +++ b/go.mod @@ -131,7 +131,7 @@ require ( github.com/Jille/raft-grpc-transport v1.5.0 github.com/arangodb/go-driver v1.6.2 github.com/armon/go-metrics v0.4.1 - github.com/aws/aws-sdk-go-v2 v1.26.1 + github.com/aws/aws-sdk-go-v2 v1.27.0 github.com/aws/aws-sdk-go-v2/config v1.27.11 github.com/aws/aws-sdk-go-v2/credentials v1.17.11 github.com/aws/aws-sdk-go-v2/service/s3 v1.53.2 diff --git a/go.sum b/go.sum index a0c312d0c..e935d5c57 100644 --- a/go.sum +++ b/go.sum @@ -142,8 +142,8 @@ github.com/armon/go-metrics v0.4.1 h1:hR91U9KYmb6bLBYLQjyM+3j+rcd/UhE+G78SFnF8gJ github.com/armon/go-metrics v0.4.1/go.mod h1:E6amYzXo6aW1tqzoZGT755KkbgrJsSdpwZ+3JqfkOG4= github.com/aws/aws-sdk-go v1.51.30 h1:RVFkjn9P0JMwnuZCVH0TlV5k9zepHzlbc4943eZMhGw= github.com/aws/aws-sdk-go v1.51.30/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk= -github.com/aws/aws-sdk-go-v2 v1.26.1 h1:5554eUqIYVWpU0YmeeYZ0wU64H2VLBs8TlhRB2L+EkA= -github.com/aws/aws-sdk-go-v2 v1.26.1/go.mod h1:ffIFB97e2yNsv4aTSGkqtHnppsIJzw7G7BReUZ3jCXM= +github.com/aws/aws-sdk-go-v2 v1.27.0 h1:7bZWKoXhzI+mMR/HjdMx8ZCC5+6fY0lS5tr0bbgiLlo= +github.com/aws/aws-sdk-go-v2 v1.27.0/go.mod h1:ffIFB97e2yNsv4aTSGkqtHnppsIJzw7G7BReUZ3jCXM= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.2 h1:x6xsQXGSmW6frevwDA+vi/wqhp1ct18mVXYN08/93to= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.2/go.mod h1:lPprDr1e6cJdyYeGXnRaJoP4Md+cDBvi2eOj00BlGmg= github.com/aws/aws-sdk-go-v2/config v1.27.11 h1:f47rANd2LQEYHda2ddSCKYId18/8BhSRM4BULGmfgNA= From 2be58ea76784196a392b84b57d7d43595e68d273 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 20 May 2024 09:05:50 -0700 Subject: [PATCH 22/24] chore(deps): bump cloud.google.com/go/storage from 1.40.0 to 1.41.0 (#5607) Bumps [cloud.google.com/go/storage](https://github.com/googleapis/google-cloud-go) from 1.40.0 to 1.41.0. - [Release notes](https://github.com/googleapis/google-cloud-go/releases) - [Changelog](https://github.com/googleapis/google-cloud-go/blob/main/CHANGES.md) - [Commits](https://github.com/googleapis/google-cloud-go/compare/spanner/v1.40.0...spanner/v1.41.0) --- updated-dependencies: - dependency-name: cloud.google.com/go/storage dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 5 +++-- go.sum | 9 +++++---- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 1672e8e62..fbeb8b034 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.22.0 require ( cloud.google.com/go v0.113.0 // indirect cloud.google.com/go/pubsub v1.38.0 - cloud.google.com/go/storage v1.40.0 + cloud.google.com/go/storage v1.41.0 github.com/Azure/azure-pipeline-go v0.2.3 github.com/Azure/azure-storage-blob-go v0.15.0 github.com/Shopify/sarama v1.38.1 @@ -162,7 +162,7 @@ require ( cloud.google.com/go/auth v0.4.1 // indirect cloud.google.com/go/auth/oauth2adapt v0.2.2 // indirect cloud.google.com/go/compute/metadata v0.3.0 // indirect - cloud.google.com/go/iam v1.1.7 // indirect + cloud.google.com/go/iam v1.1.8 // indirect filippo.io/edwards25519 v1.1.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/azcore v1.10.0 // indirect github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.5.1 // indirect @@ -276,6 +276,7 @@ require ( github.com/pelletier/go-toml/v2 v2.1.0 // indirect github.com/pengsrc/go-shared v0.2.1-0.20190131101655-1999055a4a14 // indirect github.com/philhofer/fwd v1.1.2 // indirect + github.com/pierrec/lz4/v4 v4.1.18 // indirect github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c // indirect github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c // indirect github.com/pingcap/kvproto v0.0.0-20230403051650-e166ae588106 // indirect diff --git a/go.sum b/go.sum index e935d5c57..4b59e8a5b 100644 --- a/go.sum +++ b/go.sum @@ -30,8 +30,8 @@ cloud.google.com/go/compute/metadata v0.3.0 h1:Tz+eQXMEqDIKRsmY3cHTL6FVaynIjX2Qx cloud.google.com/go/compute/metadata v0.3.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= -cloud.google.com/go/iam v1.1.7 h1:z4VHOhwKLF/+UYXAJDFwGtNF0b6gjsW1Pk9Ml0U/IoM= -cloud.google.com/go/iam v1.1.7/go.mod h1:J4PMPg8TtyurAUvSmPj8FF3EDgY1SPRZxcUGrn7WXGA= +cloud.google.com/go/iam v1.1.8 h1:r7umDwhj+BQyz0ScZMp4QrGXjSTI3ZINnpgU2nlB/K0= +cloud.google.com/go/iam v1.1.8/go.mod h1:GvE6lyMmfxXauzNq8NbgJbeVQNspG+tcdL/W8QO1+zE= cloud.google.com/go/kms v1.15.8 h1:szIeDCowID8th2i8XE4uRev5PMxQFqW+JjwYxL9h6xs= cloud.google.com/go/kms v1.15.8/go.mod h1:WoUHcDjD9pluCg7pNds131awnH429QGvRM3N/4MyoVs= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= @@ -45,8 +45,8 @@ cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0Zeo cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= -cloud.google.com/go/storage v1.40.0 h1:VEpDQV5CJxFmJ6ueWNsKxcr1QAYOXEgxDa+sBbJahPw= -cloud.google.com/go/storage v1.40.0/go.mod h1:Rrj7/hKlG87BLqDJYtwR0fbPld8uJPbQ2ucUMY7Ir0g= +cloud.google.com/go/storage v1.41.0 h1:RusiwatSu6lHeEXe3kglxakAmAbfV+rhtPqA6i8RBx0= +cloud.google.com/go/storage v1.41.0/go.mod h1:J1WCa/Z2FcgdEDuPUY8DxT5I+d9mFKsCepp5vR6Sq80= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= @@ -737,6 +737,7 @@ github.com/peterh/liner v1.2.2/go.mod h1:xFwJyiKIXJZUKItq5dGHZSTBRAuG/CpeNpWLyiN github.com/philhofer/fwd v1.1.2 h1:bnDivRJ1EWPjUIRXV5KfORO897HTbpFAQddBdE8t7Gw= github.com/philhofer/fwd v1.1.2/go.mod h1:qkPdfjR2SIEbspLqpe1tO4n5yICnr2DY7mqEx2tUTP0= github.com/pierrec/lz4/v4 v4.1.18 h1:xaKrnTkyoqfh1YItXl56+6KJNVYWlEEPuAQW9xsplYQ= +github.com/pierrec/lz4/v4 v4.1.18/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pingcap/errors v0.11.0/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c h1:xpW9bvK+HuuTmyFqUwr+jcCvpVkK7sumiz+ko5H9eq4= From 2f474c33d0f650d43ecced7017388b6b567a400b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 20 May 2024 09:05:57 -0700 Subject: [PATCH 23/24] chore(deps): bump github.com/aws/aws-sdk-go from 1.51.30 to 1.53.5 (#5608) Bumps [github.com/aws/aws-sdk-go](https://github.com/aws/aws-sdk-go) from 1.51.30 to 1.53.5. - [Release notes](https://github.com/aws/aws-sdk-go/releases) - [Commits](https://github.com/aws/aws-sdk-go/compare/v1.51.30...v1.53.5) --- updated-dependencies: - dependency-name: github.com/aws/aws-sdk-go dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index fbeb8b034..5f385531b 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/Azure/azure-pipeline-go v0.2.3 github.com/Azure/azure-storage-blob-go v0.15.0 github.com/Shopify/sarama v1.38.1 - github.com/aws/aws-sdk-go v1.51.30 + github.com/aws/aws-sdk-go v1.53.5 github.com/beorn7/perks v1.0.1 // indirect github.com/bwmarrin/snowflake v0.3.0 github.com/cenkalti/backoff/v4 v4.3.0 diff --git a/go.sum b/go.sum index 4b59e8a5b..297fd9c36 100644 --- a/go.sum +++ b/go.sum @@ -140,8 +140,8 @@ github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878/go.mod h1:3AMJUQh github.com/armon/go-metrics v0.3.9/go.mod h1:4O98XIr/9W0sxpJ8UaYkvjk10Iff7SnFrb4QAOwNTFc= github.com/armon/go-metrics v0.4.1 h1:hR91U9KYmb6bLBYLQjyM+3j+rcd/UhE+G78SFnF8gJA= github.com/armon/go-metrics v0.4.1/go.mod h1:E6amYzXo6aW1tqzoZGT755KkbgrJsSdpwZ+3JqfkOG4= -github.com/aws/aws-sdk-go v1.51.30 h1:RVFkjn9P0JMwnuZCVH0TlV5k9zepHzlbc4943eZMhGw= -github.com/aws/aws-sdk-go v1.51.30/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk= +github.com/aws/aws-sdk-go v1.53.5 h1:1OcVWMjGlwt7EU5OWmmEEXqaYfmX581EK317QJZXItM= +github.com/aws/aws-sdk-go v1.53.5/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk= github.com/aws/aws-sdk-go-v2 v1.27.0 h1:7bZWKoXhzI+mMR/HjdMx8ZCC5+6fY0lS5tr0bbgiLlo= github.com/aws/aws-sdk-go-v2 v1.27.0/go.mod h1:ffIFB97e2yNsv4aTSGkqtHnppsIJzw7G7BReUZ3jCXM= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.2 h1:x6xsQXGSmW6frevwDA+vi/wqhp1ct18mVXYN08/93to= From d218fe54faa9e8603024c6957685d6c60941ef09 Mon Sep 17 00:00:00 2001 From: chrislu Date: Mon, 20 May 2024 11:03:56 -0700 Subject: [PATCH 24/24] go fmt --- weed/mount/page_writer/page_chunk_mem.go | 2 +- weed/mount/page_writer/page_chunk_swapfile.go | 2 +- weed/mq/broker/broker_grpc_configure.go | 1 - weed/mq/broker/broker_grpc_pub.go | 5 +-- weed/mq/broker/broker_grpc_pub_follow.go | 9 ++--- .../broker_topic_partition_read_write.go | 2 +- .../mq/client/cmd/weed_pub_kv/publisher_kv.go | 2 +- .../cmd/weed_pub_record/publisher_record.go | 2 +- weed/mq/client/pub_client/publish.go | 2 +- .../sub_client/connect_to_sub_coordinator.go | 2 +- weed/mq/pub_balancer/broker_stats.go | 6 +-- weed/mq/schema/schema.go | 6 +-- weed/mq/schema/schema_builder.go | 6 +-- weed/mq/schema/schema_test.go | 40 +++++++++---------- weed/mq/schema/struct_to_schema_test.go | 4 +- weed/mq/schema/to_parquet_levels.go | 16 ++++---- weed/mq/schema/to_parquet_levels_test.go | 10 ++--- weed/mq/schema/to_parquet_schema.go | 1 - weed/mq/schema/to_parquet_value.go | 2 +- weed/mq/schema/to_schema_value.go | 16 ++++---- weed/mq/schema/write_parquet_test.go | 8 ++-- weed/mq/topic/local_partition.go | 16 ++++---- weed/shell/command_volume_balance_test.go | 2 +- weed/topology/volume_growth.go | 19 ++++----- weed/topology/volume_layout.go | 2 +- weed/util/log_buffer/log_buffer_test.go | 4 +- 26 files changed, 92 insertions(+), 95 deletions(-) diff --git a/weed/mount/page_writer/page_chunk_mem.go b/weed/mount/page_writer/page_chunk_mem.go index ddb192b9b..b5c8f9ebd 100644 --- a/weed/mount/page_writer/page_chunk_mem.go +++ b/weed/mount/page_writer/page_chunk_mem.go @@ -67,7 +67,7 @@ func (mc *MemChunk) ReadDataAt(p []byte, off int64, tsNs int64) (maxStop int64) maxStop = max(maxStop, logicStop) if t.TsNs >= tsNs { - println("read new data1", t.TsNs - tsNs, "ns") + println("read new data1", t.TsNs-tsNs, "ns") } } } diff --git a/weed/mount/page_writer/page_chunk_swapfile.go b/weed/mount/page_writer/page_chunk_swapfile.go index 35e9ee682..350821757 100644 --- a/weed/mount/page_writer/page_chunk_swapfile.go +++ b/weed/mount/page_writer/page_chunk_swapfile.go @@ -137,7 +137,7 @@ func (sc *SwapFileChunk) ReadDataAt(p []byte, off int64, tsNs int64) (maxStop in maxStop = max(maxStop, logicStop) if t.TsNs >= tsNs { - println("read new data2", t.TsNs - tsNs, "ns") + println("read new data2", t.TsNs-tsNs, "ns") } } } diff --git a/weed/mq/broker/broker_grpc_configure.go b/weed/mq/broker/broker_grpc_configure.go index 40ac8df23..afb01a886 100644 --- a/weed/mq/broker/broker_grpc_configure.go +++ b/weed/mq/broker/broker_grpc_configure.go @@ -35,7 +35,6 @@ func (b *MessageQueueBroker) ConfigureTopic(ctx context.Context, request *mq_pb. } } - t := topic.FromPbTopic(request.Topic) var readErr, assignErr error resp, readErr = b.readTopicConfFromFiler(t) diff --git a/weed/mq/broker/broker_grpc_pub.go b/weed/mq/broker/broker_grpc_pub.go index a217489de..d633a3efa 100644 --- a/weed/mq/broker/broker_grpc_pub.go +++ b/weed/mq/broker/broker_grpc_pub.go @@ -69,7 +69,7 @@ func (b *MessageQueueBroker) PublishMessage(stream mq_pb.SeaweedMessaging_Publis return stream.Send(response) } - var receivedSequence, acknowledgedSequence int64 + var receivedSequence, acknowledgedSequence int64 var isClosed bool // start sending ack to publisher @@ -85,7 +85,7 @@ func (b *MessageQueueBroker) PublishMessage(stream mq_pb.SeaweedMessaging_Publis lastAckTime := time.Now() for !isClosed { receivedSequence = atomic.LoadInt64(&localTopicPartition.AckTsNs) - if acknowledgedSequence < receivedSequence && (receivedSequence - acknowledgedSequence >= ackInterval || time.Since(lastAckTime) > 1*time.Second){ + if acknowledgedSequence < receivedSequence && (receivedSequence-acknowledgedSequence >= ackInterval || time.Since(lastAckTime) > 1*time.Second) { acknowledgedSequence = receivedSequence response := &mq_pb.PublishMessageResponse{ AckSequence: acknowledgedSequence, @@ -101,7 +101,6 @@ func (b *MessageQueueBroker) PublishMessage(stream mq_pb.SeaweedMessaging_Publis } }() - // process each published messages clientName := fmt.Sprintf("%v-%4d/%s/%v", findClientAddress(stream.Context()), rand.Intn(10000), initMessage.Topic, initMessage.Partition) localTopicPartition.Publishers.AddPublisher(clientName, topic.NewLocalPublisher()) diff --git a/weed/mq/broker/broker_grpc_pub_follow.go b/weed/mq/broker/broker_grpc_pub_follow.go index d8100f021..8995b0cc2 100644 --- a/weed/mq/broker/broker_grpc_pub_follow.go +++ b/weed/mq/broker/broker_grpc_pub_follow.go @@ -13,10 +13,11 @@ import ( ) type memBuffer struct { - buf []byte - startTime time.Time - stopTime time.Time + buf []byte + startTime time.Time + stopTime time.Time } + func (b *MessageQueueBroker) PublishFollowMe(stream mq_pb.SeaweedMessaging_PublishFollowMeServer) (err error) { var req *mq_pb.PublishFollowMeRequest req, err = stream.Recv() @@ -84,7 +85,6 @@ func (b *MessageQueueBroker) PublishFollowMe(stream mq_pb.SeaweedMessaging_Publi } } - t, p := topic.FromPbTopic(initMessage.Topic), topic.FromPbPartition(initMessage.Partition) logBuffer.ShutdownLogBuffer() @@ -97,7 +97,6 @@ func (b *MessageQueueBroker) PublishFollowMe(stream mq_pb.SeaweedMessaging_Publi partitionGeneration := time.Unix(0, p.UnixTimeNs).UTC().Format(topic.TIME_FORMAT) partitionDir := fmt.Sprintf("%s/%s/%04d-%04d", topicDir, partitionGeneration, p.RangeStart, p.RangeStop) - // flush the remaining messages inMemoryBuffers.CloseInput() for mem, found := inMemoryBuffers.Dequeue(); found; mem, found = inMemoryBuffers.Dequeue() { diff --git a/weed/mq/broker/broker_topic_partition_read_write.go b/weed/mq/broker/broker_topic_partition_read_write.go index 50470f879..cb55e2032 100644 --- a/weed/mq/broker/broker_topic_partition_read_write.go +++ b/weed/mq/broker/broker_topic_partition_read_write.go @@ -45,7 +45,7 @@ func (b *MessageQueueBroker) genLogFlushFunc(t topic.Topic, partition *mq_pb.Par b.accessLock.Lock() defer b.accessLock.Unlock() p := topic.FromPbPartition(partition) - if localPartition:=b.localTopicManager.GetLocalPartition(t, p); localPartition!=nil { + if localPartition := b.localTopicManager.GetLocalPartition(t, p); localPartition != nil { localPartition.NotifyLogFlushed(logBuffer.LastFlushTsNs) } diff --git a/weed/mq/client/cmd/weed_pub_kv/publisher_kv.go b/weed/mq/client/cmd/weed_pub_kv/publisher_kv.go index 59842c21f..096b355a1 100644 --- a/weed/mq/client/cmd/weed_pub_kv/publisher_kv.go +++ b/weed/mq/client/cmd/weed_pub_kv/publisher_kv.go @@ -16,7 +16,7 @@ var ( concurrency = flag.Int("c", 4, "concurrent publishers") partitionCount = flag.Int("p", 6, "partition count") - clientName = flag.String("client", "c1", "client name") + clientName = flag.String("client", "c1", "client name") namespace = flag.String("ns", "test", "namespace") t = flag.String("t", "test", "t") diff --git a/weed/mq/client/cmd/weed_pub_record/publisher_record.go b/weed/mq/client/cmd/weed_pub_record/publisher_record.go index 4d4e5fc4d..a5fbd455e 100644 --- a/weed/mq/client/cmd/weed_pub_record/publisher_record.go +++ b/weed/mq/client/cmd/weed_pub_record/publisher_record.go @@ -19,7 +19,7 @@ var ( concurrency = flag.Int("c", 4, "concurrent publishers") partitionCount = flag.Int("p", 6, "partition count") - clientName = flag.String("client", "c1", "client name") + clientName = flag.String("client", "c1", "client name") namespace = flag.String("ns", "test", "namespace") t = flag.String("t", "test", "t") diff --git a/weed/mq/client/pub_client/publish.go b/weed/mq/client/pub_client/publish.go index 0c162d6a0..a25620de1 100644 --- a/weed/mq/client/pub_client/publish.go +++ b/weed/mq/client/pub_client/publish.go @@ -48,7 +48,7 @@ func (p *TopicPublisher) FinishPublish() error { if inputBuffers, found := p.partition2Buffer.AllIntersections(0, pub_balancer.MaxPartitionCount); found { for _, inputBuffer := range inputBuffers { inputBuffer.Enqueue(&mq_pb.DataMessage{ - TsNs: time.Now().UnixNano(), + TsNs: time.Now().UnixNano(), Ctrl: &mq_pb.ControlMessage{ IsClose: true, }, diff --git a/weed/mq/client/sub_client/connect_to_sub_coordinator.go b/weed/mq/client/sub_client/connect_to_sub_coordinator.go index b0b533e42..2f1330b5e 100644 --- a/weed/mq/client/sub_client/connect_to_sub_coordinator.go +++ b/weed/mq/client/sub_client/connect_to_sub_coordinator.go @@ -122,7 +122,7 @@ func (sub *TopicSubscriber) onEachPartition(assigned *mq_pb.BrokerPartitionAssig StartTsNs: sub.alreadyProcessedTsNs, StartType: mq_pb.PartitionOffsetStartType_EARLIEST_IN_MEMORY, }, - Filter: sub.ContentConfig.Filter, + Filter: sub.ContentConfig.Filter, FollowerBrokers: assigned.FollowerBrokers, }, }, diff --git a/weed/mq/pub_balancer/broker_stats.go b/weed/mq/pub_balancer/broker_stats.go index 00f1f80ca..c579c275e 100644 --- a/weed/mq/pub_balancer/broker_stats.go +++ b/weed/mq/pub_balancer/broker_stats.go @@ -17,7 +17,7 @@ type BrokerStats struct { } type TopicPartitionStats struct { topic.TopicPartition - PublisherCount int32 + PublisherCount int32 SubscriberCount int32 } @@ -48,7 +48,7 @@ func (bs *BrokerStats) UpdateStats(stats *mq_pb.BrokerStats) { UnixTimeNs: topicPartitionStats.Partition.UnixTimeNs, }, }, - PublisherCount: topicPartitionStats.PublisherCount, + PublisherCount: topicPartitionStats.PublisherCount, SubscriberCount: topicPartitionStats.SubscriberCount, } publisherCount += topicPartitionStats.PublisherCount @@ -76,7 +76,7 @@ func (bs *BrokerStats) RegisterAssignment(t *mq_pb.Topic, partition *mq_pb.Parti UnixTimeNs: partition.UnixTimeNs, }, }, - PublisherCount: 0, + PublisherCount: 0, SubscriberCount: 0, } key := tps.TopicPartition.String() diff --git a/weed/mq/schema/schema.go b/weed/mq/schema/schema.go index 1f99dda5b..5fadf2cc2 100644 --- a/weed/mq/schema/schema.go +++ b/weed/mq/schema/schema.go @@ -6,17 +6,17 @@ import ( type Schema struct { RecordType *schema_pb.RecordType - fieldMap map[string]*schema_pb.Field + fieldMap map[string]*schema_pb.Field } func NewSchema(recordType *schema_pb.RecordType) (*Schema, error) { - fieldMap := make( map[string]*schema_pb.Field) + fieldMap := make(map[string]*schema_pb.Field) for _, field := range recordType.Fields { fieldMap[field.Name] = field } return &Schema{ RecordType: recordType, - fieldMap: fieldMap, + fieldMap: fieldMap, }, nil } diff --git a/weed/mq/schema/schema_builder.go b/weed/mq/schema/schema_builder.go index c64f8bb7c..db89ce34c 100644 --- a/weed/mq/schema/schema_builder.go +++ b/weed/mq/schema/schema_builder.go @@ -8,9 +8,9 @@ import ( var ( TypeBoolean = &schema_pb.Type{Kind: &schema_pb.Type_ScalarType{schema_pb.ScalarType_BOOL}} TypeInt32 = &schema_pb.Type{Kind: &schema_pb.Type_ScalarType{schema_pb.ScalarType_INT32}} - TypeInt64 = &schema_pb.Type{Kind: &schema_pb.Type_ScalarType{schema_pb.ScalarType_INT64}} - TypeFloat = &schema_pb.Type{Kind: &schema_pb.Type_ScalarType{schema_pb.ScalarType_FLOAT}} - TypeDouble = &schema_pb.Type{Kind: &schema_pb.Type_ScalarType{schema_pb.ScalarType_DOUBLE}} + TypeInt64 = &schema_pb.Type{Kind: &schema_pb.Type_ScalarType{schema_pb.ScalarType_INT64}} + TypeFloat = &schema_pb.Type{Kind: &schema_pb.Type_ScalarType{schema_pb.ScalarType_FLOAT}} + TypeDouble = &schema_pb.Type{Kind: &schema_pb.Type_ScalarType{schema_pb.ScalarType_DOUBLE}} TypeBytes = &schema_pb.Type{Kind: &schema_pb.Type_ScalarType{schema_pb.ScalarType_BYTES}} TypeString = &schema_pb.Type{Kind: &schema_pb.Type_ScalarType{schema_pb.ScalarType_STRING}} ) diff --git a/weed/mq/schema/schema_test.go b/weed/mq/schema/schema_test.go index 5398beb02..f7dc8ff55 100644 --- a/weed/mq/schema/schema_test.go +++ b/weed/mq/schema/schema_test.go @@ -32,10 +32,10 @@ func TestEnumScalarType(t *testing.T) { func TestField(t *testing.T) { field := &Field{ - Name: "field_name", - Type: &Type{Kind: &Type_ScalarType{ScalarType: ScalarType_INT32}}, - FieldIndex: 1, - IsRepeated: false, + Name: "field_name", + Type: &Type{Kind: &Type_ScalarType{ScalarType: ScalarType_INT32}}, + FieldIndex: 1, + IsRepeated: false, } assert.NotNil(t, field) } @@ -44,32 +44,32 @@ func TestRecordType(t *testing.T) { subRecord := &RecordType{ Fields: []*Field{ { - Name: "field_1", - Type: &Type{Kind: &Type_ScalarType{ScalarType: ScalarType_INT32}}, - FieldIndex: 1, - IsRepeated: false, + Name: "field_1", + Type: &Type{Kind: &Type_ScalarType{ScalarType: ScalarType_INT32}}, + FieldIndex: 1, + IsRepeated: false, }, { - Name: "field_2", - Type: &Type{Kind: &Type_ScalarType{ScalarType: ScalarType_STRING}}, - FieldIndex: 2, - IsRepeated: false, + Name: "field_2", + Type: &Type{Kind: &Type_ScalarType{ScalarType: ScalarType_STRING}}, + FieldIndex: 2, + IsRepeated: false, }, }, } record := &RecordType{ Fields: []*Field{ { - Name: "field_key", - Type: &Type{Kind: &Type_ScalarType{ScalarType: ScalarType_INT32}}, - FieldIndex: 1, - IsRepeated: false, + Name: "field_key", + Type: &Type{Kind: &Type_ScalarType{ScalarType: ScalarType_INT32}}, + FieldIndex: 1, + IsRepeated: false, }, { - Name: "field_record", - Type: &Type{Kind: &Type_RecordType{RecordType: subRecord}}, - FieldIndex: 2, - IsRepeated: false, + Name: "field_record", + Type: &Type{Kind: &Type_RecordType{RecordType: subRecord}}, + FieldIndex: 2, + IsRepeated: false, }, }, } diff --git a/weed/mq/schema/struct_to_schema_test.go b/weed/mq/schema/struct_to_schema_test.go index d22939aef..fae27ecef 100644 --- a/weed/mq/schema/struct_to_schema_test.go +++ b/weed/mq/schema/struct_to_schema_test.go @@ -76,7 +76,7 @@ func TestStructToSchema(t *testing.T) { RecordTypeBegin(). WithField("Field3", TypeString). WithField("Field4", TypeInt32). - RecordTypeEnd(), + RecordTypeEnd(), ). RecordTypeEnd(), }, @@ -104,7 +104,7 @@ func TestStructToSchema(t *testing.T) { RecordTypeBegin(). WithField("Field6", TypeString). WithField("Field7", TypeBytes). - RecordTypeEnd(), + RecordTypeEnd(), ).RecordTypeEnd(), ). RecordTypeEnd(), diff --git a/weed/mq/schema/to_parquet_levels.go b/weed/mq/schema/to_parquet_levels.go index 6c73563cd..f9fc073bb 100644 --- a/weed/mq/schema/to_parquet_levels.go +++ b/weed/mq/schema/to_parquet_levels.go @@ -7,9 +7,9 @@ import ( type ParquetLevels struct { startColumnIndex int - endColumnIndex int - definitionDepth int - levels map[string]*ParquetLevels + endColumnIndex int + definitionDepth int + levels map[string]*ParquetLevels } func ToParquetLevels(recordType *schema_pb.RecordType) (*ParquetLevels, error) { @@ -19,7 +19,7 @@ func ToParquetLevels(recordType *schema_pb.RecordType) (*ParquetLevels, error) { func toFieldTypeLevels(fieldType *schema_pb.Type, startColumnIndex, definitionDepth int) (*ParquetLevels, error) { switch fieldType.Kind.(type) { case *schema_pb.Type_ScalarType: - return toFieldTypeScalarLevels(fieldType.GetScalarType(), startColumnIndex, definitionDepth) + return toFieldTypeScalarLevels(fieldType.GetScalarType(), startColumnIndex, definitionDepth) case *schema_pb.Type_RecordType: return toRecordTypeLevels(fieldType.GetRecordType(), startColumnIndex, definitionDepth) case *schema_pb.Type_ListType: @@ -35,15 +35,15 @@ func toFieldTypeListLevels(listType *schema_pb.ListType, startColumnIndex, defin func toFieldTypeScalarLevels(scalarType schema_pb.ScalarType, startColumnIndex, definitionDepth int) (*ParquetLevels, error) { return &ParquetLevels{ startColumnIndex: startColumnIndex, - endColumnIndex: startColumnIndex + 1, - definitionDepth: definitionDepth, + endColumnIndex: startColumnIndex + 1, + definitionDepth: definitionDepth, }, nil } func toRecordTypeLevels(recordType *schema_pb.RecordType, startColumnIndex, definitionDepth int) (*ParquetLevels, error) { recordTypeLevels := &ParquetLevels{ startColumnIndex: startColumnIndex, - definitionDepth: definitionDepth, - levels: make(map[string]*ParquetLevels), + definitionDepth: definitionDepth, + levels: make(map[string]*ParquetLevels), } for _, field := range recordType.Fields { fieldTypeLevels, err := toFieldTypeLevels(field.Type, startColumnIndex, definitionDepth+1) diff --git a/weed/mq/schema/to_parquet_levels_test.go b/weed/mq/schema/to_parquet_levels_test.go index 150bfb146..5200c0e02 100644 --- a/weed/mq/schema/to_parquet_levels_test.go +++ b/weed/mq/schema/to_parquet_levels_test.go @@ -11,9 +11,9 @@ func TestToParquetLevels(t *testing.T) { recordType *schema_pb.RecordType } tests := []struct { - name string - args args - want *ParquetLevels + name string + args args + want *ParquetLevels }{ { name: "nested type", @@ -25,13 +25,13 @@ func TestToParquetLevels(t *testing.T) { RecordTypeBegin(). WithField("zName", TypeString). WithField("emails", ListOf(TypeString)). - RecordTypeEnd()). + RecordTypeEnd()). WithField("Company", TypeString). WithRecordField("Address", RecordTypeBegin(). WithField("Street", TypeString). WithField("City", TypeString). - RecordTypeEnd()). + RecordTypeEnd()). RecordTypeEnd(), }, want: &ParquetLevels{ diff --git a/weed/mq/schema/to_parquet_schema.go b/weed/mq/schema/to_parquet_schema.go index 077305948..196546a32 100644 --- a/weed/mq/schema/to_parquet_schema.go +++ b/weed/mq/schema/to_parquet_schema.go @@ -31,7 +31,6 @@ func toParquetFieldType(fieldType *schema_pb.Type) (dataType parquet.Node, err e return nil, fmt.Errorf("unknown field type: %T", fieldType.Kind) } - return dataType, err } diff --git a/weed/mq/schema/to_parquet_value.go b/weed/mq/schema/to_parquet_value.go index 22a93de67..83740495b 100644 --- a/weed/mq/schema/to_parquet_value.go +++ b/weed/mq/schema/to_parquet_value.go @@ -70,7 +70,7 @@ func doVisitValue(fieldType *schema_pb.Type, levels *ParquetLevels, fieldValue * return } -func toParquetValue(value *schema_pb.Value) (parquet.Value, error) { +func toParquetValue(value *schema_pb.Value) (parquet.Value, error) { switch value.Kind.(type) { case *schema_pb.Value_BoolValue: return parquet.BooleanValue(value.GetBoolValue()), nil diff --git a/weed/mq/schema/to_schema_value.go b/weed/mq/schema/to_schema_value.go index 93103cd1a..947a84310 100644 --- a/weed/mq/schema/to_schema_value.go +++ b/weed/mq/schema/to_schema_value.go @@ -47,7 +47,7 @@ func toRecordValue(recordType *schema_pb.RecordType, levels *ParquetLevels, valu func toListValue(listType *schema_pb.ListType, levels *ParquetLevels, values []parquet.Value, valueIndex int) (listValue *schema_pb.Value, endValueIndex int, err error) { listValues := make([]*schema_pb.Value, 0) var value *schema_pb.Value - for ;valueIndex < len(values); { + for valueIndex < len(values) { if values[valueIndex].Column() != levels.startColumnIndex { break } @@ -67,19 +67,19 @@ func toScalarValue(scalarType schema_pb.ScalarType, levels *ParquetLevels, value } switch scalarType { case schema_pb.ScalarType_BOOL: - return &schema_pb.Value{Kind: &schema_pb.Value_BoolValue{BoolValue: value.Boolean()}}, valueIndex+1, nil + return &schema_pb.Value{Kind: &schema_pb.Value_BoolValue{BoolValue: value.Boolean()}}, valueIndex + 1, nil case schema_pb.ScalarType_INT32: - return &schema_pb.Value{Kind: &schema_pb.Value_Int32Value{Int32Value: value.Int32()}}, valueIndex+1, nil + return &schema_pb.Value{Kind: &schema_pb.Value_Int32Value{Int32Value: value.Int32()}}, valueIndex + 1, nil case schema_pb.ScalarType_INT64: - return &schema_pb.Value{Kind: &schema_pb.Value_Int64Value{Int64Value: value.Int64()}}, valueIndex+1, nil + return &schema_pb.Value{Kind: &schema_pb.Value_Int64Value{Int64Value: value.Int64()}}, valueIndex + 1, nil case schema_pb.ScalarType_FLOAT: - return &schema_pb.Value{Kind: &schema_pb.Value_FloatValue{FloatValue: value.Float()}}, valueIndex+1, nil + return &schema_pb.Value{Kind: &schema_pb.Value_FloatValue{FloatValue: value.Float()}}, valueIndex + 1, nil case schema_pb.ScalarType_DOUBLE: - return &schema_pb.Value{Kind: &schema_pb.Value_DoubleValue{DoubleValue: value.Double()}}, valueIndex+1, nil + return &schema_pb.Value{Kind: &schema_pb.Value_DoubleValue{DoubleValue: value.Double()}}, valueIndex + 1, nil case schema_pb.ScalarType_BYTES: - return &schema_pb.Value{Kind: &schema_pb.Value_BytesValue{BytesValue: value.ByteArray()}}, valueIndex+1, nil + return &schema_pb.Value{Kind: &schema_pb.Value_BytesValue{BytesValue: value.ByteArray()}}, valueIndex + 1, nil case schema_pb.ScalarType_STRING: - return &schema_pb.Value{Kind: &schema_pb.Value_StringValue{StringValue: string(value.ByteArray())}}, valueIndex+1, nil + return &schema_pb.Value{Kind: &schema_pb.Value_StringValue{StringValue: string(value.ByteArray())}}, valueIndex + 1, nil } return nil, valueIndex, fmt.Errorf("unsupported scalar type: %v", scalarType) } diff --git a/weed/mq/schema/write_parquet_test.go b/weed/mq/schema/write_parquet_test.go index e5df8ecd9..5cfdec999 100644 --- a/weed/mq/schema/write_parquet_test.go +++ b/weed/mq/schema/write_parquet_test.go @@ -19,13 +19,13 @@ func TestWriteReadParquet(t *testing.T) { RecordTypeBegin(). WithField("zName", TypeString). WithField("emails", ListOf(TypeString)). - RecordTypeEnd()). + RecordTypeEnd()). WithField("Company", TypeString). WithRecordField("Address", RecordTypeBegin(). WithField("Street", TypeString). WithField("City", TypeString). - RecordTypeEnd()). + RecordTypeEnd()). RecordTypeEnd() fmt.Printf("RecordType: %v\n", recordType) @@ -85,9 +85,9 @@ func testWritingParquetFile(t *testing.T, count int, filename string, parquetSch fmt.Sprintf("john_%d@c.com", i), fmt.Sprintf("john_%d@d.com", i), fmt.Sprintf("john_%d@e.com", i)). - RecordEnd()). + RecordEnd()). SetString("Company", fmt.Sprintf("company_%d", i)). - RecordEnd() + RecordEnd() AddRecordValue(rowBuilder, recordType, parquetLevels, recordValue) if count < 10 { diff --git a/weed/mq/topic/local_partition.go b/weed/mq/topic/local_partition.go index 54c122a0f..72e78d606 100644 --- a/weed/mq/topic/local_partition.go +++ b/weed/mq/topic/local_partition.go @@ -16,17 +16,17 @@ import ( ) type LocalPartition struct { - ListenersWaits int64 - AckTsNs int64 + ListenersWaits int64 + AckTsNs int64 // notifying clients ListenersLock sync.Mutex ListenersCond *sync.Cond Partition - LogBuffer *log_buffer.LogBuffer - Publishers *LocalPartitionPublishers - Subscribers *LocalPartitionSubscribers + LogBuffer *log_buffer.LogBuffer + Publishers *LocalPartitionPublishers + Subscribers *LocalPartitionSubscribers followerStream mq_pb.SeaweedMessaging_PublishFollowMeClient followerGrpcConnection *grpc.ClientConn @@ -37,7 +37,7 @@ var TIME_FORMAT = "2006-01-02-15-04-05" func NewLocalPartition(partition Partition, logFlushFn log_buffer.LogFlushFuncType, readFromDiskFn log_buffer.LogReadFromDiskFuncType) *LocalPartition { lp := &LocalPartition{ - Partition: partition, + Partition: partition, Publishers: NewLocalPartitionPublishers(), Subscribers: NewLocalPartitionSubscribers(), } @@ -155,8 +155,8 @@ func (p *LocalPartition) MaybeConnectToFollowers(initMessage *mq_pb.PublishMessa if err = p.followerStream.Send(&mq_pb.PublishFollowMeRequest{ Message: &mq_pb.PublishFollowMeRequest_Init{ Init: &mq_pb.PublishFollowMeRequest_InitMessage{ - Topic: initMessage.Topic, - Partition: initMessage.Partition, + Topic: initMessage.Topic, + Partition: initMessage.Partition, }, }, }); err != nil { diff --git a/weed/shell/command_volume_balance_test.go b/weed/shell/command_volume_balance_test.go index b8af4fb98..fb39e063f 100644 --- a/weed/shell/command_volume_balance_test.go +++ b/weed/shell/command_volume_balance_test.go @@ -278,7 +278,7 @@ func TestDeleteEmptySelection(t *testing.T) { eachDataNode(topologyInfo, func(dc string, rack RackId, dn *master_pb.DataNodeInfo) { for _, diskInfo := range dn.DiskInfos { for _, v := range diskInfo.VolumeInfos { - if v.Size <= super_block.SuperBlockSize && v.ModifiedAtSecond > 0 { + if v.Size <= super_block.SuperBlockSize && v.ModifiedAtSecond > 0 { fmt.Printf("empty volume %d from %s\n", v.Id, dn.Id) } } diff --git a/weed/topology/volume_growth.go b/weed/topology/volume_growth.go index 9885fc2d1..44b8b6286 100644 --- a/weed/topology/volume_growth.go +++ b/weed/topology/volume_growth.go @@ -31,20 +31,20 @@ type VolumeGrowRequest struct { } type volumeGrowthStrategy struct { - Copy1Count int - Copy2Count int - Copy3Count int + Copy1Count int + Copy2Count int + Copy3Count int CopyOtherCount int - Threshold float64 + Threshold float64 } var ( VolumeGrowStrategy = volumeGrowthStrategy{ - Copy1Count: 7, - Copy2Count: 6, - Copy3Count: 3, + Copy1Count: 7, + Copy2Count: 6, + Copy3Count: 3, CopyOtherCount: 1, - Threshold: 0.9, + Threshold: 0.9, } ) @@ -77,7 +77,8 @@ func NewDefaultVolumeGrowth() *VolumeGrowth { // given copyCount, how many logical volumes to create func (vg *VolumeGrowth) findVolumeCount(copyCount int) (count int) { switch copyCount { - case 1: count = VolumeGrowStrategy.Copy1Count + case 1: + count = VolumeGrowStrategy.Copy1Count case 2: count = VolumeGrowStrategy.Copy2Count case 3: diff --git a/weed/topology/volume_layout.go b/weed/topology/volume_layout.go index 4467b2dc8..66f7118c9 100644 --- a/weed/topology/volume_layout.go +++ b/weed/topology/volume_layout.go @@ -381,7 +381,7 @@ func (vl *VolumeLayout) GetActiveVolumeCount(option *VolumeGrowOption) (total, a } active++ info, _ := dn.GetVolumesById(v) - if float64(info.Size) > float64(vl.volumeSizeLimit)* VolumeGrowStrategy.Threshold{ + if float64(info.Size) > float64(vl.volumeSizeLimit)*VolumeGrowStrategy.Threshold { crowded++ } } diff --git a/weed/util/log_buffer/log_buffer_test.go b/weed/util/log_buffer/log_buffer_test.go index 067a02ef4..a4947a611 100644 --- a/weed/util/log_buffer/log_buffer_test.go +++ b/weed/util/log_buffer/log_buffer_test.go @@ -19,7 +19,7 @@ func TestNewLogBufferFirstBuffer(t *testing.T) { }, nil, func() { }) - startTime := MessagePosition{Time:time.Now()} + startTime := MessagePosition{Time: time.Now()} messageSize := 1024 messageCount := 5000 @@ -38,7 +38,7 @@ func TestNewLogBufferFirstBuffer(t *testing.T) { println("processed all messages") return true, io.EOF } - return false,nil + return false, nil }) fmt.Printf("before flush: sent %d received %d\n", messageCount, receivedMessageCount)