Merge branch 'master' into list_recursive_prefixed_entries

8 months ago · 2477fadd2c
16 changed files with 1070 additions and 89 deletions
--- a/go.mod
+++ b/go.mod
@ -34,7 +34,6 @@ require (
 	github.com/go-sql-driver/mysql v1.8.1
 	github.com/go-zookeeper/zk v1.0.3 // indirect
 	github.com/gocql/gocql v1.6.0
-	github.com/golang-jwt/jwt v3.2.2+incompatible // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/protobuf v1.5.4
 	github.com/golang/snappy v0.0.4 // indirect
@ -75,7 +74,7 @@ require (
 	github.com/prometheus/client_golang v1.19.0
 	github.com/prometheus/client_model v0.5.0 // indirect
 	github.com/prometheus/common v0.48.0 // indirect
-	github.com/prometheus/procfs v0.13.0
+	github.com/prometheus/procfs v0.15.0
 	github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect
 	github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
 	github.com/seaweedfs/goexif v1.0.3
@ -133,8 +132,8 @@ require (
 	github.com/armon/go-metrics v0.4.1
 	github.com/aws/aws-sdk-go-v2 v1.27.0
 	github.com/aws/aws-sdk-go-v2/config v1.27.11
-	github.com/aws/aws-sdk-go-v2/credentials v1.17.11
-	github.com/aws/aws-sdk-go-v2/service/s3 v1.53.2
+	github.com/aws/aws-sdk-go-v2/credentials v1.17.16
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.54.3
 	github.com/cognusion/imaging v1.0.1
 	github.com/fluent/fluent-logger-golang v1.9.0
 	github.com/getsentry/sentry-go v0.27.0
@ -151,8 +150,8 @@ require (
 	github.com/schollz/progressbar/v3 v3.14.2
 	github.com/shirou/gopsutil/v3 v3.24.4
 	github.com/tikv/client-go/v2 v2.0.7
-	github.com/ydb-platform/ydb-go-sdk-auth-environ v0.2.0
-	github.com/ydb-platform/ydb-go-sdk/v3 v3.66.3
+	github.com/ydb-platform/ydb-go-sdk-auth-environ v0.4.2
+	github.com/ydb-platform/ydb-go-sdk/v3 v3.68.1
 	go.etcd.io/etcd/client/pkg/v3 v3.5.13
 	go.uber.org/atomic v1.11.0
 	google.golang.org/grpc/security/advancedtls v0.0.0-20220622233350-5cdb09fa29c1
@ -186,20 +185,20 @@ require (
 	github.com/andybalholm/cascadia v1.3.2 // indirect
 	github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e // indirect
 	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.2 // indirect
-	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.1 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.5 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.5 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.3 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.7 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.7 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.5 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.7 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.7 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.7 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.5 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.9 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.9 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.7 // indirect
 	github.com/aws/aws-sdk-go-v2/service/sns v1.29.2 // indirect
 	github.com/aws/aws-sdk-go-v2/service/sqs v1.31.2 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sso v1.20.5 // indirect
-	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.4 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sts v1.28.6 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.20.9 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.24.3 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.28.10 // indirect
 	github.com/aws/smithy-go v1.20.2 // indirect
 	github.com/boltdb/bolt v1.3.1 // indirect
 	github.com/bradenaw/juniper v0.15.2 // indirect
@ -310,8 +309,8 @@ require (
 	github.com/xanzy/ssh-agent v0.3.3 // indirect
 	github.com/yandex-cloud/go-genproto v0.0.0-20211115083454-9ca41db5ed9e // indirect
 	github.com/ydb-platform/ydb-go-genproto v0.0.0-20240316140903-4a47abca1cca // indirect
-	github.com/ydb-platform/ydb-go-yc v0.10.2 // indirect
-	github.com/ydb-platform/ydb-go-yc-metadata v0.5.2 // indirect
+	github.com/ydb-platform/ydb-go-yc v0.12.1 // indirect
+	github.com/ydb-platform/ydb-go-yc-metadata v0.6.1 // indirect
 	github.com/yunify/qingstor-sdk-go/v3 v3.2.0 // indirect
 	github.com/yusufpapurcu/wmi v1.2.4 // indirect
 	github.com/zeebo/blake3 v0.2.3 // indirect
--- a/go.sum
+++ b/go.sum
--- a/k8s/charts/seaweedfs/Chart.yaml
+++ b/k8s/charts/seaweedfs/Chart.yaml
@ -2,4 +2,4 @@ apiVersion: v1
 description: SeaweedFS
 name: seaweedfs
 appVersion: "3.67"
-version: 3.67.0
+version: 3.67.1
--- a/k8s/charts/seaweedfs/templates/filer-statefulset.yaml
+++ b/k8s/charts/seaweedfs/templates/filer-statefulset.yaml
@ -129,7 +129,7 @@ spec:
            - "-ec"
            - |
              exec /usr/bin/weed \
-              {{- if eq .Values.filer.logs.type "hostPath" }}
+              {{- if or (eq .Values.filer.logs.type "hostPath") (eq .Values.filer.logs.type "emptyDir") }}
              -logdir=/logs \
              {{- else }}
              -logtostderr=true \
@ -197,7 +197,7 @@ spec:
              {{- end }}
              -master={{ if .Values.global.masterServer }}{{.Values.global.masterServer}}{{ else }}{{ range $index := until (.Values.master.replicas | int) }}${SEAWEEDFS_FULLNAME}-master-{{ $index }}.${SEAWEEDFS_FULLNAME}-master.{{ $.Release.Namespace }}:{{ $.Values.master.port }}{{ if lt $index (sub ($.Values.master.replicas | int) 1) }},{{ end }}{{ end }}{{ end }}
          volumeMounts:
-            {{- if eq .Values.filer.logs.type "hostPath" }}
+            {{- if or (eq .Values.filer.logs.type "hostPath") (eq .Values.filer.logs.type "emptyDir") }}
            - name: seaweedfs-filer-log-volume
              mountPath: "/logs/"
            {{- end }}
@ -206,7 +206,7 @@ spec:
              mountPath: /etc/sw
              readOnly: true
            {{- end }}
-            {{- if (or .Values.filer.enablePVC (or (eq .Values.filer.data.type "hostPath") (eq .Values.filer.data.type "persistentVolumeClaim"))) }}
+            {{- if (or .Values.filer.enablePVC (or (eq .Values.filer.data.type "hostPath") (eq .Values.filer.data.type "persistentVolumeClaim") (eq .Values.filer.data.type "emptyDir"))) }}
            - name: data-filer
              mountPath: /data
            {{- end }}
@ -285,6 +285,10 @@ spec:
          persistentVolumeClaim:
            claimName: {{ .Values.filer.logs.claimName }}
        {{- end }}
+        {{- if eq .Values.filer.logs.type "emptyDir" }}
+        - name: seaweedfs-filer-log-volume
+          emptyDir: {}
+        {{- end }}
        {{- if eq .Values.filer.data.type "hostPath" }}
        - name: data-filer
          hostPath:
@ -296,6 +300,10 @@ spec:
          persistentVolumeClaim:
            claimName: {{ .Values.filer.data.claimName }}
        {{- end }}
+        {{- if eq .Values.filer.data.type "emptyDir" }}
+        - name: data-filer
+          emptyDir: {}
+        {{- end }}
        - name: db-schema-config-volume
          configMap:
            name: seaweedfs-db-init-config
--- a/k8s/charts/seaweedfs/templates/master-statefulset.yaml
+++ b/k8s/charts/seaweedfs/templates/master-statefulset.yaml
@ -108,9 +108,9 @@ spec:
          command:
            - "/bin/sh"
            - "-ec"
-            - | 
+            - |
              exec /usr/bin/weed \
-              {{- if eq .Values.master.logs.type "hostPath" }}
+              {{- if or (eq .Values.master.logs.type "hostPath") (eq .Values.master.logs.type "emptyDir") }}
              -logdir=/logs \
              {{- else }}
              -logtostderr=true \
@ -158,7 +158,7 @@ spec:
          volumeMounts:
            - name : data-{{ .Release.Namespace }}
              mountPath: /data
-            {{- if eq .Values.master.logs.type "hostPath" }}
+            {{- if or (eq .Values.master.logs.type "hostPath") (eq .Values.master.logs.type "emptyDir") }}
            - name: seaweedfs-master-log-volume
              mountPath: "/logs/"
            {{- end }}
@ -241,7 +241,11 @@ spec:
        {{- if eq .Values.master.logs.type "existingClaim" }}
        - name: seaweedfs-master-log-volume
          persistentVolumeClaim:
-            claimName: {{ .Values.master.logs.claimName }} 
+            claimName: {{ .Values.master.logs.claimName }}
+        {{- end }}
+        {{- if eq .Values.master.logs.type "emptyDir" }}
+        - name: seaweedfs-master-log-volume
+          emptyDir: {}
        {{- end }}
        {{- if eq .Values.master.data.type "hostPath" }}
        - name: data-{{ .Release.Namespace }}
@ -252,7 +256,11 @@ spec:
        {{- if eq .Values.master.data.type "existingClaim" }}
        - name: data-{{ .Release.Namespace }}
          persistentVolumeClaim:
-            claimName: {{ .Values.master.data.claimName }} 
+            claimName: {{ .Values.master.data.claimName }}
+        {{- end }}
+        {{- if eq .Values.master.data.type "emptyDir" }}
+        - name: data-{{ .Release.Namespace }}
+          emptyDir: {}
        {{- end }}
        - name: master-config
          configMap:
--- a/k8s/charts/seaweedfs/templates/s3-deployment.yaml
+++ b/k8s/charts/seaweedfs/templates/s3-deployment.yaml
@ -99,7 +99,7 @@ spec:
            - "-ec"
            - |
              exec /usr/bin/weed \
-               {{- if eq .Values.s3.logs.type "hostPath" }}
+              {{- if or (eq .Values.s3.logs.type "hostPath") (eq .Values.s3.logs.type "emptyDir") }}
              -logdir=/logs \
              {{- else }}
              -logtostderr=true \
@ -136,7 +136,7 @@ spec:
              {{- end }}
              -filer={{ template "seaweedfs.name" . }}-filer-client.{{ .Release.Namespace }}:{{ .Values.filer.port }}
          volumeMounts:
-            {{- if eq .Values.s3.logs.type "hostPath" }}
+            {{- if or (eq .Values.s3.logs.type "hostPath") (eq .Values.s3.logs.type "emptyDir") }}
            - name: logs
              mountPath: "/logs/"
            {{- end }}
@ -225,6 +225,10 @@ spec:
            path: {{ .Values.s3.logs.hostPathPrefix }}/logs/seaweedfs/s3
            type: DirectoryOrCreate
        {{- end }}
+        {{- if eq .Values.s3.logs.type "emptyDir" }}
+        - name: logs
+          emptyDir: {}
+        {{- end }}
        {{- if .Values.global.enableSecurity }}
        - name: security-config
          configMap:
--- a/k8s/charts/seaweedfs/templates/volume-statefulset.yaml
+++ b/k8s/charts/seaweedfs/templates/volume-statefulset.yaml
@ -261,6 +261,10 @@ spec:
          persistentVolumeClaim:
            claimName: {{ $dir.claimName }}
      {{- end }}
+      {{- if eq $dir.type "emptyDir" }}
+        - name: {{ $dir.name }}
+          emptyDir: {}
+      {{- end }}

     {{- end }}

@ -276,6 +280,10 @@ spec:
          persistentVolumeClaim:
            claimName: {{ .Values.volume.idx.claimName }}
       {{- end }}
+       {{- if eq .Values.volume.idx.type "emptyDir" }}
+        - name: idx
+          emptyDir: {}
+       {{- end }}
     {{- end }}

     {{- if .Values.volume.logs }}
@ -290,6 +298,10 @@ spec:
          persistentVolumeClaim:
            claimName: {{ .Values.volume.logs.claimName }}
       {{- end }}
+       {{- if eq .Values.volume.logs.type "emptyDir" }}
+        - name: logs
+          emptyDir: {}
+       {{- end }}
     {{- end }}
     {{- if .Values.global.enableSecurity }}
        - name: security-config
--- a/k8s/charts/seaweedfs/values.yaml
+++ b/k8s/charts/seaweedfs/values.yaml
@ -93,11 +93,18 @@ master:
  #  data:
  #    type: "existingClaim"
  #    claimName: "my-pvc"
+  #
+  # You can also use emptyDir storage:
+  #  data:
+  #    type: "emptyDir"
  data:
    type: "hostPath"
    storageClass: ""
    hostPathPrefix: /ssd

+  # You can also use emptyDir storage:
+  #  logs:
+  #    type: "emptyDir"
  logs:
    type: "hostPath"
    size: ""
@ -275,9 +282,14 @@ volume:
  #     maxVolumes: 0  # If set to zero on non-windows OS, the limit will be auto configured. (default "7")
  #
  # You may also spacify an existing claim:
-  #    - name: data
-  #      type: "existingClaim"
-  #      claimName: "my-pvc"
+  #   - name: data
+  #     type: "existingClaim"
+  #     claimName: "my-pvc"
+  #     maxVolumes: 0  # If set to zero on non-windows OS, the limit will be auto configured. (default "7")
+  #
+  # You can also use emptyDir storage:
+  #   - name: data
+  #     type: "emptyDir"
  #     maxVolumes: 0  # If set to zero on non-windows OS, the limit will be auto configured. (default "7")

  dataDirs:
@ -310,6 +322,11 @@ volume:
  # idx:
  #   type: "existingClaim"
  #   claimName: "myClaim"
+  #
+  # or
+  #
+  # idx:
+  #  type: "emptyDir"

  # same applies to "logs"

@ -499,12 +516,19 @@ filer:
  #  data:
  #    type: "existingClaim"
  #    claimName: "my-pvc"
+  #
+  # You can also use emptyDir storage:
+  #  data:
+  #    type: "emptyDir"
  data:
    type: "hostPath"
    size: ""
    storageClass: ""
    hostPathPrefix: /storage

+  # You can also use emptyDir storage:
+  #  logs:
+  #    type: "emptyDir"
  logs:
    type: "hostPath"
    size: ""
@ -741,7 +765,7 @@ s3:
  # should map directly to the value of the resources field for a PodSpec,
  # formatted as a multi-line string. By default no direct resource request
  # is made.
-  resources: {}
+  resources: null

  # Toleration Settings for server pods
  # This should be a multi-line string matching the Toleration array
@ -783,6 +807,9 @@ s3:
  #   allowPrivilegeEscalation: false
  containerSecurityContext: {}

+  # You can also use emptyDir storage:
+  #  logs:
+  #    type: "emptyDir"
  logs:
    type: "hostPath"
    size: ""
--- a/unmaintained/stress_filer_upload/bench_filer_upload/bench_filer_upload.go
+++ b/unmaintained/stress_filer_upload/bench_filer_upload/bench_filer_upload.go
@ -66,6 +66,7 @@ func main() {

 	go func() {
 		ticker := time.NewTicker(1000 * time.Millisecond)
+		defer ticker.Stop()

 		var lastTime time.Time
 		var counter, size int64
--- a/unmaintained/stress_filer_upload/stress_filer_upload_actual/stress_filer_upload.go
+++ b/unmaintained/stress_filer_upload/stress_filer_upload_actual/stress_filer_upload.go
@ -72,6 +72,7 @@ func main() {

 	go func() {
 		ticker := time.NewTicker(500 * time.Millisecond)
+		defer ticker.Stop()

 		var lastTime time.Time
 		var counter, size int64
--- a/weed/filer/ydb/ydb_store.go
+++ b/weed/filer/ydb/ydb_store.go
@ -79,7 +79,7 @@ func (store *YdbStore) initialize(dirBuckets string, dsn string, tablePathPrefix
 	}
 	opts := []ydb.Option{
 		ydb.WithDialTimeout(time.Duration(dialTimeOut) * time.Second),
-		environ.WithEnvironCredentials(ctx),
+		environ.WithEnvironCredentials(),
 	}
 	if poolSizeLimit > 0 {
 		opts = append(opts, ydb.WithSessionPoolSizeLimit(poolSizeLimit))
--- a/weed/replication/sink/s3sink/s3_sink.go
+++ b/weed/replication/sink/s3sink/s3_sink.go
@ -1,6 +1,7 @@
 package S3Sink

 import (
+	"encoding/base64"
 	"fmt"
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/credentials"
@ -11,7 +12,6 @@ import (
 	"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
 	"strconv"
 	"strings"
-	"encoding/base64"

 	"github.com/seaweedfs/seaweedfs/weed/filer"
 	"github.com/seaweedfs/seaweedfs/weed/glog"
--- a/weed/server/master_grpc_server.go
+++ b/weed/server/master_grpc_server.go
@ -297,6 +297,7 @@ func (ms *MasterServer) KeepConnected(stream master_pb.Seaweed_KeepConnectedServ
 	}()

 	ticker := time.NewTicker(5 * time.Second)
+	defer ticker.Stop()
 	for {
 		select {
 		case message := <-messageChan:
--- a/weed/server/volume_grpc_client_to_master.go
+++ b/weed/server/volume_grpc_client_to_master.go
@ -159,7 +159,9 @@ func (vs *VolumeServer) doHeartbeat(masterAddress pb.ServerAddress, grpcDialOpti
 	}

 	volumeTickChan := time.NewTicker(sleepInterval)
+	defer volumeTickChan.Stop()
 	ecShardTickChan := time.NewTicker(17 * sleepInterval)
+	defer ecShardTickChan.Stop()
 	dataCenter := vs.store.GetDataCenter()
 	rack := vs.store.GetRack()
 	ip := vs.store.Ip
--- a/weed/shell/command_volume_server_leave.go
+++ b/weed/shell/command_volume_server_leave.go
@ -25,7 +25,7 @@ func (c *commandVolumeServerLeave) Name() string {
 func (c *commandVolumeServerLeave) Help() string {
 	return `stop a volume server from sending heartbeats to the master

-	volume.unmount -node <volume server host:port> -force
+	volumeServer.leave -node <volume server host:port> -force

 	This command enables gracefully shutting down the volume server.
 	The volume server will stop sending heartbeats to the master.
--- a/weed/stats/disk_openbsd.go
+++ b/weed/stats/disk_openbsd.go
@ -4,22 +4,21 @@
 package stats

 import (
-    "syscall"
+	"syscall"

-    "github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
 )

 func fillInDiskStatus(disk *volume_server_pb.DiskStatus) {
-    fs := syscall.Statfs_t{}
-    err := syscall.Statfs(disk.Dir, &fs)
-    if err != nil {
-        return
-    }
-    disk.All = fs.F_blocks * uint64(fs.F_bsize)
-    disk.Free = fs.F_bfree * uint64(fs.F_bsize)
-    disk.Used = disk.All - disk.Free
-    disk.PercentFree = float32((float64(disk.Free) / float64(disk.All)) * 100)
-    disk.PercentUsed = float32((float64(disk.Used) / float64(disk.All)) * 100)
-    return
+	fs := syscall.Statfs_t{}
+	err := syscall.Statfs(disk.Dir, &fs)
+	if err != nil {
+		return
+	}
+	disk.All = fs.F_blocks * uint64(fs.F_bsize)
+	disk.Free = fs.F_bfree * uint64(fs.F_bsize)
+	disk.Used = disk.All - disk.Free
+	disk.PercentFree = float32((float64(disk.Free) / float64(disk.All)) * 100)
+	disk.PercentUsed = float32((float64(disk.Used) / float64(disk.All)) * 100)
+	return
 }
-