Merge branch 'master' into list_recursive_prefixed_entries

go.mod

@@ -34,7 +34,6 @@ require (
 	github.com/go-sql-driver/mysql v1.8.1
 	github.com/go-zookeeper/zk v1.0.3 // indirect
 	github.com/gocql/gocql v1.6.0
-	github.com/golang-jwt/jwt v3.2.2+incompatible // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/protobuf v1.5.4
 	github.com/golang/snappy v0.0.4 // indirect
@@ -75,7 +74,7 @@ require (
 	github.com/prometheus/client_golang v1.19.0
 	github.com/prometheus/client_model v0.5.0 // indirect
 	github.com/prometheus/common v0.48.0 // indirect
-	github.com/prometheus/procfs v0.13.0
+	github.com/prometheus/procfs v0.15.0
 	github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect
 	github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
 	github.com/seaweedfs/goexif v1.0.3
@@ -133,8 +132,8 @@ require (
 	github.com/armon/go-metrics v0.4.1
 	github.com/aws/aws-sdk-go-v2 v1.27.0
 	github.com/aws/aws-sdk-go-v2/config v1.27.11
-	github.com/aws/aws-sdk-go-v2/credentials v1.17.11
-	github.com/aws/aws-sdk-go-v2/service/s3 v1.53.2
+	github.com/aws/aws-sdk-go-v2/credentials v1.17.16
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.54.3
 	github.com/cognusion/imaging v1.0.1
 	github.com/fluent/fluent-logger-golang v1.9.0
 	github.com/getsentry/sentry-go v0.27.0
@@ -151,8 +150,8 @@ require (
 	github.com/schollz/progressbar/v3 v3.14.2
 	github.com/shirou/gopsutil/v3 v3.24.4
 	github.com/tikv/client-go/v2 v2.0.7
-	github.com/ydb-platform/ydb-go-sdk-auth-environ v0.2.0
-	github.com/ydb-platform/ydb-go-sdk/v3 v3.66.3
+	github.com/ydb-platform/ydb-go-sdk-auth-environ v0.4.2
+	github.com/ydb-platform/ydb-go-sdk/v3 v3.68.1
 	go.etcd.io/etcd/client/pkg/v3 v3.5.13
 	go.uber.org/atomic v1.11.0
 	google.golang.org/grpc/security/advancedtls v0.0.0-20220622233350-5cdb09fa29c1
@@ -186,20 +185,20 @@ require (
 	github.com/andybalholm/cascadia v1.3.2 // indirect
 	github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e // indirect
 	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.2 // indirect
-	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.1 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.5 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.5 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.3 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.7 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.7 // indirect
 	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.5 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.7 // indirect
 	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.7 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.7 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.5 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.9 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.9 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.7 // indirect
 	github.com/aws/aws-sdk-go-v2/service/sns v1.29.2 // indirect
 	github.com/aws/aws-sdk-go-v2/service/sqs v1.31.2 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sso v1.20.5 // indirect
-	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.23.4 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sts v1.28.6 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.20.9 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.24.3 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.28.10 // indirect
 	github.com/aws/smithy-go v1.20.2 // indirect
 	github.com/boltdb/bolt v1.3.1 // indirect
 	github.com/bradenaw/juniper v0.15.2 // indirect
@@ -310,8 +309,8 @@ require (
 	github.com/xanzy/ssh-agent v0.3.3 // indirect
 	github.com/yandex-cloud/go-genproto v0.0.0-20211115083454-9ca41db5ed9e // indirect
 	github.com/ydb-platform/ydb-go-genproto v0.0.0-20240316140903-4a47abca1cca // indirect
-	github.com/ydb-platform/ydb-go-yc v0.10.2 // indirect
-	github.com/ydb-platform/ydb-go-yc-metadata v0.5.2 // indirect
+	github.com/ydb-platform/ydb-go-yc v0.12.1 // indirect
+	github.com/ydb-platform/ydb-go-yc-metadata v0.6.1 // indirect
 	github.com/yunify/qingstor-sdk-go/v3 v3.2.0 // indirect
 	github.com/yusufpapurcu/wmi v1.2.4 // indirect
 	github.com/zeebo/blake3 v0.2.3 // indirect

k8s/charts/seaweedfs/Chart.yaml

@@ -2,4 +2,4 @@ apiVersion: v1
 description: SeaweedFS
 name: seaweedfs
 appVersion: "3.67"
-version: 3.67.0
+version: 3.67.1

k8s/charts/seaweedfs/templates/filer-statefulset.yaml

@@ -129,7 +129,7 @@ spec:
             - "-ec"
             - |
               exec /usr/bin/weed \
-              {{- if eq .Values.filer.logs.type "hostPath" }}
+              {{- if or (eq .Values.filer.logs.type "hostPath") (eq .Values.filer.logs.type "emptyDir") }}
               -logdir=/logs \
               {{- else }}
               -logtostderr=true \
@@ -197,7 +197,7 @@ spec:
               {{- end }}
               -master={{ if .Values.global.masterServer }}{{.Values.global.masterServer}}{{ else }}{{ range $index := until (.Values.master.replicas | int) }}${SEAWEEDFS_FULLNAME}-master-{{ $index }}.${SEAWEEDFS_FULLNAME}-master.{{ $.Release.Namespace }}:{{ $.Values.master.port }}{{ if lt $index (sub ($.Values.master.replicas | int) 1) }},{{ end }}{{ end }}{{ end }}
           volumeMounts:
-            {{- if eq .Values.filer.logs.type "hostPath" }}
+            {{- if or (eq .Values.filer.logs.type "hostPath") (eq .Values.filer.logs.type "emptyDir") }}
             - name: seaweedfs-filer-log-volume
               mountPath: "/logs/"
             {{- end }}
@@ -206,7 +206,7 @@ spec:
               mountPath: /etc/sw
               readOnly: true
             {{- end }}
-            {{- if (or .Values.filer.enablePVC (or (eq .Values.filer.data.type "hostPath") (eq .Values.filer.data.type "persistentVolumeClaim"))) }}
+            {{- if (or .Values.filer.enablePVC (or (eq .Values.filer.data.type "hostPath") (eq .Values.filer.data.type "persistentVolumeClaim") (eq .Values.filer.data.type "emptyDir"))) }}
             - name: data-filer
               mountPath: /data
             {{- end }}
@@ -285,6 +285,10 @@ spec:
           persistentVolumeClaim:
             claimName: {{ .Values.filer.logs.claimName }}
         {{- end }}
+        {{- if eq .Values.filer.logs.type "emptyDir" }}
+        - name: seaweedfs-filer-log-volume
+          emptyDir: {}
+        {{- end }}
         {{- if eq .Values.filer.data.type "hostPath" }}
         - name: data-filer
           hostPath:
@@ -296,6 +300,10 @@ spec:
           persistentVolumeClaim:
             claimName: {{ .Values.filer.data.claimName }}
         {{- end }}
+        {{- if eq .Values.filer.data.type "emptyDir" }}
+        - name: data-filer
+          emptyDir: {}
+        {{- end }}
         - name: db-schema-config-volume
           configMap:
             name: seaweedfs-db-init-config

k8s/charts/seaweedfs/templates/master-statefulset.yaml

@@ -110,7 +110,7 @@ spec:
             - "-ec"
             - |
               exec /usr/bin/weed \
-              {{- if eq .Values.master.logs.type "hostPath" }}
+              {{- if or (eq .Values.master.logs.type "hostPath") (eq .Values.master.logs.type "emptyDir") }}
               -logdir=/logs \
               {{- else }}
               -logtostderr=true \
@@ -158,7 +158,7 @@ spec:
           volumeMounts:
             - name : data-{{ .Release.Namespace }}
               mountPath: /data
-            {{- if eq .Values.master.logs.type "hostPath" }}
+            {{- if or (eq .Values.master.logs.type "hostPath") (eq .Values.master.logs.type "emptyDir") }}
             - name: seaweedfs-master-log-volume
               mountPath: "/logs/"
             {{- end }}
@@ -243,6 +243,10 @@ spec:
           persistentVolumeClaim:
             claimName: {{ .Values.master.logs.claimName }}
         {{- end }}
+        {{- if eq .Values.master.logs.type "emptyDir" }}
+        - name: seaweedfs-master-log-volume
+          emptyDir: {}
+        {{- end }}
         {{- if eq .Values.master.data.type "hostPath" }}
         - name: data-{{ .Release.Namespace }}
           hostPath:
@@ -254,6 +258,10 @@ spec:
           persistentVolumeClaim:
             claimName: {{ .Values.master.data.claimName }}
         {{- end }}
+        {{- if eq .Values.master.data.type "emptyDir" }}
+        - name: data-{{ .Release.Namespace }}
+          emptyDir: {}
+        {{- end }}
         - name: master-config
           configMap:
             name: {{ template "seaweedfs.name" . }}-master-config

k8s/charts/seaweedfs/templates/s3-deployment.yaml

@@ -99,7 +99,7 @@ spec:
             - "-ec"
             - |
               exec /usr/bin/weed \
-               {{- if eq .Values.s3.logs.type "hostPath" }}
+              {{- if or (eq .Values.s3.logs.type "hostPath") (eq .Values.s3.logs.type "emptyDir") }}
               -logdir=/logs \
               {{- else }}
               -logtostderr=true \
@@ -136,7 +136,7 @@ spec:
               {{- end }}
               -filer={{ template "seaweedfs.name" . }}-filer-client.{{ .Release.Namespace }}:{{ .Values.filer.port }}
           volumeMounts:
-            {{- if eq .Values.s3.logs.type "hostPath" }}
+            {{- if or (eq .Values.s3.logs.type "hostPath") (eq .Values.s3.logs.type "emptyDir") }}
             - name: logs
               mountPath: "/logs/"
             {{- end }}
@@ -225,6 +225,10 @@ spec:
             path: {{ .Values.s3.logs.hostPathPrefix }}/logs/seaweedfs/s3
             type: DirectoryOrCreate
         {{- end }}
+        {{- if eq .Values.s3.logs.type "emptyDir" }}
+        - name: logs
+          emptyDir: {}
+        {{- end }}
         {{- if .Values.global.enableSecurity }}
         - name: security-config
           configMap:

k8s/charts/seaweedfs/templates/volume-statefulset.yaml

@@ -261,6 +261,10 @@ spec:
           persistentVolumeClaim:
             claimName: {{ $dir.claimName }}
       {{- end }}
+      {{- if eq $dir.type "emptyDir" }}
+        - name: {{ $dir.name }}
+          emptyDir: {}
+      {{- end }}
 
      {{- end }}
 
@@ -276,6 +280,10 @@ spec:
           persistentVolumeClaim:
             claimName: {{ .Values.volume.idx.claimName }}
        {{- end }}
+       {{- if eq .Values.volume.idx.type "emptyDir" }}
+        - name: idx
+          emptyDir: {}
+       {{- end }}
      {{- end }}
 
      {{- if .Values.volume.logs }}
@@ -290,6 +298,10 @@ spec:
           persistentVolumeClaim:
             claimName: {{ .Values.volume.logs.claimName }}
        {{- end }}
+       {{- if eq .Values.volume.logs.type "emptyDir" }}
+        - name: logs
+          emptyDir: {}
+       {{- end }}
      {{- end }}
      {{- if .Values.global.enableSecurity }}
         - name: security-config

k8s/charts/seaweedfs/values.yaml

@@ -93,11 +93,18 @@ master:
   #  data:
   #    type: "existingClaim"
   #    claimName: "my-pvc"
+  #
+  # You can also use emptyDir storage:
+  #  data:
+  #    type: "emptyDir"
   data:
     type: "hostPath"
     storageClass: ""
     hostPathPrefix: /ssd
 
+  # You can also use emptyDir storage:
+  #  logs:
+  #    type: "emptyDir"
   logs:
     type: "hostPath"
     size: ""
@@ -279,6 +286,11 @@ volume:
   #     type: "existingClaim"
   #     claimName: "my-pvc"
   #     maxVolumes: 0  # If set to zero on non-windows OS, the limit will be auto configured. (default "7")
+  #
+  # You can also use emptyDir storage:
+  #   - name: data
+  #     type: "emptyDir"
+  #     maxVolumes: 0  # If set to zero on non-windows OS, the limit will be auto configured. (default "7")
 
   dataDirs:
   - name: data1
@@ -310,6 +322,11 @@ volume:
   # idx:
   #   type: "existingClaim"
   #   claimName: "myClaim"
+  #
+  # or
+  #
+  # idx:
+  #  type: "emptyDir"
 
   # same applies to "logs"
 
@@ -499,12 +516,19 @@ filer:
   #  data:
   #    type: "existingClaim"
   #    claimName: "my-pvc"
+  #
+  # You can also use emptyDir storage:
+  #  data:
+  #    type: "emptyDir"
   data:
     type: "hostPath"
     size: ""
     storageClass: ""
     hostPathPrefix: /storage
 
+  # You can also use emptyDir storage:
+  #  logs:
+  #    type: "emptyDir"
   logs:
     type: "hostPath"
     size: ""
@@ -741,7 +765,7 @@ s3:
   # should map directly to the value of the resources field for a PodSpec,
   # formatted as a multi-line string. By default no direct resource request
   # is made.
-  resources: {}
+  resources: null
 
   # Toleration Settings for server pods
   # This should be a multi-line string matching the Toleration array
@@ -783,6 +807,9 @@ s3:
   #   allowPrivilegeEscalation: false
   containerSecurityContext: {}
 
+  # You can also use emptyDir storage:
+  #  logs:
+  #    type: "emptyDir"
   logs:
     type: "hostPath"
     size: ""

unmaintained/stress_filer_upload/bench_filer_upload/bench_filer_upload.go

@@ -66,6 +66,7 @@ func main() {
 
 	go func() {
 		ticker := time.NewTicker(1000 * time.Millisecond)
+		defer ticker.Stop()
 
 		var lastTime time.Time
 		var counter, size int64

unmaintained/stress_filer_upload/stress_filer_upload_actual/stress_filer_upload.go

@@ -72,6 +72,7 @@ func main() {
 
 	go func() {
 		ticker := time.NewTicker(500 * time.Millisecond)
+		defer ticker.Stop()
 
 		var lastTime time.Time
 		var counter, size int64

weed/filer/ydb/ydb_store.go

@@ -79,7 +79,7 @@ func (store *YdbStore) initialize(dirBuckets string, dsn string, tablePathPrefix
 	}
 	opts := []ydb.Option{
 		ydb.WithDialTimeout(time.Duration(dialTimeOut) * time.Second),
-		environ.WithEnvironCredentials(ctx),
+		environ.WithEnvironCredentials(),
 	}
 	if poolSizeLimit > 0 {
 		opts = append(opts, ydb.WithSessionPoolSizeLimit(poolSizeLimit))

weed/replication/sink/s3sink/s3_sink.go

@@ -1,6 +1,7 @@
 package S3Sink
 
 import (
+	"encoding/base64"
 	"fmt"
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/credentials"
@@ -11,7 +12,6 @@ import (
 	"github.com/seaweedfs/seaweedfs/weed/s3api/s3_constants"
 	"strconv"
 	"strings"
-	"encoding/base64"
 
 	"github.com/seaweedfs/seaweedfs/weed/filer"
 	"github.com/seaweedfs/seaweedfs/weed/glog"

weed/server/master_grpc_server.go

@@ -297,6 +297,7 @@ func (ms *MasterServer) KeepConnected(stream master_pb.Seaweed_KeepConnectedServ
 	}()
 
 	ticker := time.NewTicker(5 * time.Second)
+	defer ticker.Stop()
 	for {
 		select {
 		case message := <-messageChan:

weed/server/volume_grpc_client_to_master.go

@@ -159,7 +159,9 @@ func (vs *VolumeServer) doHeartbeat(masterAddress pb.ServerAddress, grpcDialOpti
 	}
 
 	volumeTickChan := time.NewTicker(sleepInterval)
+	defer volumeTickChan.Stop()
 	ecShardTickChan := time.NewTicker(17 * sleepInterval)
+	defer ecShardTickChan.Stop()
 	dataCenter := vs.store.GetDataCenter()
 	rack := vs.store.GetRack()
 	ip := vs.store.Ip

weed/shell/command_volume_server_leave.go

@@ -25,7 +25,7 @@ func (c *commandVolumeServerLeave) Name() string {
 func (c *commandVolumeServerLeave) Help() string {
 	return `stop a volume server from sending heartbeats to the master
 
-	volume.unmount -node <volume server host:port> -force
+	volumeServer.leave -node <volume server host:port> -force
 
 	This command enables gracefully shutting down the volume server.
 	The volume server will stop sending heartbeats to the master.

weed/stats/disk_openbsd.go

@@ -22,4 +22,3 @@ func fillInDiskStatus(disk *volume_server_pb.DiskStatus) {
 	disk.PercentUsed = float32((float64(disk.Used) / float64(disk.All)) * 100)
 	return
 }
-