Merge branch 'master' into _product

.github/workflows/container_dev.yml

@@ -36,7 +36,7 @@ jobs:
         uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 # v1
       -
         name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@f03ac48505955848960e80bbb68046aa35c7b9e7 # v1
+        uses: docker/setup-buildx-action@4b4e9c3e2d4531116a6f8ba8e71fc6e2cb6e6c8c # v1
         with:
           buildkitd-flags: "--debug"
       -

.github/workflows/container_latest.yml

@@ -37,7 +37,7 @@ jobs:
         uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 # v1
       -
         name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@f03ac48505955848960e80bbb68046aa35c7b9e7 # v1
+        uses: docker/setup-buildx-action@4b4e9c3e2d4531116a6f8ba8e71fc6e2cb6e6c8c # v1
         with:
           buildkitd-flags: "--debug"
       -

.github/workflows/container_release1.yml

@@ -37,7 +37,7 @@ jobs:
         uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 # v1
       -
         name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@f03ac48505955848960e80bbb68046aa35c7b9e7 # v1
+        uses: docker/setup-buildx-action@4b4e9c3e2d4531116a6f8ba8e71fc6e2cb6e6c8c # v1
       -
         name: Login to Docker Hub
         if: github.event_name != 'pull_request'

.github/workflows/container_release2.yml

@@ -38,7 +38,7 @@ jobs:
         uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 # v1
       -
         name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@f03ac48505955848960e80bbb68046aa35c7b9e7 # v1
+        uses: docker/setup-buildx-action@4b4e9c3e2d4531116a6f8ba8e71fc6e2cb6e6c8c # v1
       -
         name: Login to Docker Hub
         if: github.event_name != 'pull_request'

.github/workflows/container_release3.yml

@@ -38,7 +38,7 @@ jobs:
         uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 # v1
       -
         name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@f03ac48505955848960e80bbb68046aa35c7b9e7 # v1
+        uses: docker/setup-buildx-action@4b4e9c3e2d4531116a6f8ba8e71fc6e2cb6e6c8c # v1
       -
         name: Login to Docker Hub
         if: github.event_name != 'pull_request'

.github/workflows/container_release4.yml

@@ -37,7 +37,7 @@ jobs:
         uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 # v1
       -
         name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@f03ac48505955848960e80bbb68046aa35c7b9e7 # v1
+        uses: docker/setup-buildx-action@4b4e9c3e2d4531116a6f8ba8e71fc6e2cb6e6c8c # v1
       -
         name: Login to Docker Hub
         if: github.event_name != 'pull_request'

.github/workflows/container_release5.yml

@@ -37,7 +37,7 @@ jobs:
         uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 # v1
       -
         name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@f03ac48505955848960e80bbb68046aa35c7b9e7 # v1
+        uses: docker/setup-buildx-action@4b4e9c3e2d4531116a6f8ba8e71fc6e2cb6e6c8c # v1
       -
         name: Login to Docker Hub
         if: github.event_name != 'pull_request'

README.md

@@ -32,7 +32,7 @@ Your support will be really appreciated by me and other supporters!
 -->
 
 ### Gold Sponsors
-[![nodion](https://www.nodion.com/img/logo.svg)](https://www.nodion.com)
+[![nodion](https://raw.githubusercontent.com/seaweedfs/seaweedfs/master/note/sponsor_nodion.png)](https://www.nodion.com)
 [![piknik](https://raw.githubusercontent.com/seaweedfs/seaweedfs/master/note/piknik.png)](https://www.piknik.com)
 
 ---

go.mod

@@ -3,13 +3,14 @@ module github.com/seaweedfs/seaweedfs
 go 1.20
 
 require (
-	cloud.google.com/go v0.107.0 // indirect
+	cloud.google.com/go v0.110.0 // indirect
 	cloud.google.com/go/pubsub v1.28.0
 	cloud.google.com/go/storage v1.29.0
 	github.com/Azure/azure-pipeline-go v0.2.3
 	github.com/Azure/azure-storage-blob-go v0.15.0
 	github.com/Shopify/sarama v1.38.1
-	github.com/aws/aws-sdk-go v1.44.209
+	github.com/aws/aws-sdk-go v1.44.219
+	github.com/cenkalti/backoff/v4 v4.2.0
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/bwmarrin/snowflake v0.3.0
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
@@ -66,7 +67,7 @@ require (
 	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/mattn/go-ieproxy v0.0.9 // indirect
 	github.com/mattn/go-isatty v0.0.17 // indirect
-	github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
+	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
 	github.com/mitchellh/mapstructure v1.5.0 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
@@ -78,10 +79,10 @@ require (
 	github.com/pquerna/cachecontrol v0.1.0
 	github.com/prometheus/client_golang v1.14.0
 	github.com/prometheus/client_model v0.3.0 // indirect
-	github.com/prometheus/common v0.37.0 // indirect
+	github.com/prometheus/common v0.39.0 // indirect
 	github.com/prometheus/procfs v0.9.0
 	github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect
-	github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 // indirect
+	github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
 	github.com/seaweedfs/goexif v1.0.3
 	github.com/seaweedfs/raft v1.1.0
 	github.com/sirupsen/logrus v1.9.0 // indirect
@@ -108,39 +109,39 @@ require (
 	go.etcd.io/etcd/client/v3 v3.5.7
 	go.mongodb.org/mongo-driver v1.11.1
 	go.opencensus.io v0.24.0 // indirect
-	gocloud.dev v0.28.0
+	gocloud.dev v0.29.0
 	gocloud.dev/pubsub/natspubsub v0.28.0
-	gocloud.dev/pubsub/rabbitpubsub v0.28.0
-	golang.org/x/crypto v0.3.0 // indirect
-	golang.org/x/exp v0.0.0-20221031165847-c99f073a8326
-	golang.org/x/image v0.0.0-20200119044424-58c23975cae1
-	golang.org/x/net v0.7.0
-	golang.org/x/oauth2 v0.5.0 // indirect
-	golang.org/x/sys v0.5.0
-	golang.org/x/text v0.7.0 // indirect
+	gocloud.dev/pubsub/rabbitpubsub v0.29.0
+	golang.org/x/crypto v0.6.0 // indirect
+	golang.org/x/exp v0.0.0-20230124195608-d38c7dcee874
+	golang.org/x/image v0.6.0
+	golang.org/x/net v0.8.0
+	golang.org/x/oauth2 v0.6.0 // indirect
+	golang.org/x/sys v0.6.0
+	golang.org/x/text v0.8.0 // indirect
 	golang.org/x/tools v0.6.0
 	golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
-	google.golang.org/api v0.110.0
+	google.golang.org/api v0.112.0
 	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/genproto v0.0.0-20230209215440-0dfe4f8abfcc // indirect
+	google.golang.org/genproto v0.0.0-20230303212802-e74f57abe488 // indirect
 	google.golang.org/grpc v1.53.0
 	google.golang.org/protobuf v1.28.1
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	modernc.org/b v1.0.0 // indirect
 	modernc.org/cc/v3 v3.40.0 // indirect
 	modernc.org/ccgo/v3 v3.16.13 // indirect
-	modernc.org/libc v1.22.2 // indirect
+	modernc.org/libc v1.22.3 // indirect
 	modernc.org/mathutil v1.5.0 // indirect
-	modernc.org/memory v1.4.0 // indirect
+	modernc.org/memory v1.5.0 // indirect
 	modernc.org/opt v0.1.3 // indirect
-	modernc.org/sqlite v1.20.4
+	modernc.org/sqlite v1.21.0
 	modernc.org/strutil v1.1.3
 	modernc.org/token v1.0.1 // indirect
 )
 
 require (
 	github.com/Jille/raft-grpc-transport v1.4.0
-	github.com/arangodb/go-driver v1.4.1
+	github.com/arangodb/go-driver v1.5.2
 	github.com/armon/go-metrics v0.4.1
 	github.com/fluent/fluent-logger-golang v1.9.0
 	github.com/google/flatbuffers/go v0.0.0-20230108230133-3b8644d32c50
@@ -151,31 +152,32 @@ require (
 	github.com/schollz/progressbar/v3 v3.13.0
 	github.com/tikv/client-go/v2 v2.0.5
 	github.com/ydb-platform/ydb-go-sdk-auth-environ v0.1.2
-	github.com/ydb-platform/ydb-go-sdk/v3 v3.42.10
+	github.com/ydb-platform/ydb-go-sdk/v3 v3.43.0
 	google.golang.org/grpc/security/advancedtls v0.0.0-20220622233350-5cdb09fa29c1
 )
 
 require (
 	cloud.google.com/go/compute v1.18.0 // indirect
 	cloud.google.com/go/compute/metadata v0.2.3 // indirect
-	cloud.google.com/go/iam v0.8.0 // indirect
+	cloud.google.com/go/iam v0.12.0 // indirect
 	github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e // indirect
-	github.com/aws/aws-sdk-go-v2 v1.17.1 // indirect
-	github.com/aws/aws-sdk-go-v2/config v1.18.3 // indirect
-	github.com/aws/aws-sdk-go-v2/credentials v1.13.3 // indirect
-	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.19 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.25 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.19 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/ini v1.3.26 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.19 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sns v1.18.6 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sqs v1.19.15 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sso v1.11.25 // indirect
-	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.8 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sts v1.17.5 // indirect
-	github.com/aws/smithy-go v1.13.4 // indirect
-	github.com/benbjohnson/clock v1.1.0 // indirect
+	github.com/aws/aws-sdk-go-v2 v1.17.4 // indirect
+	github.com/aws/aws-sdk-go-v2/config v1.18.12 // indirect
+	github.com/aws/aws-sdk-go-v2/credentials v1.13.12 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.22 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.28 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.22 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/ini v1.3.29 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.22 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sns v1.20.2 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sqs v1.20.2 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.12.1 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.1 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.18.3 // indirect
+	github.com/aws/smithy-go v1.13.5 // indirect
+	github.com/benbjohnson/clock v1.3.0 // indirect
 	github.com/boltdb/bolt v1.3.1 // indirect
+	github.com/cenkalti/backoff/v4 v4.2.0
 	github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 // indirect
 	github.com/d4l3k/messagediff v1.2.1 // indirect
 	github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 // indirect
@@ -188,13 +190,13 @@ require (
 	github.com/hashicorp/go-hclog v1.2.0 // indirect
 	github.com/hashicorp/go-immutable-radix v1.3.1 // indirect
 	github.com/hashicorp/go-msgpack v1.1.5 // indirect
-	github.com/hashicorp/golang-lru v0.5.4 // indirect
+	github.com/hashicorp/golang-lru v0.6.0 // indirect
 	github.com/jcmturner/aescts/v2 v2.0.0 // indirect
 	github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect
 	github.com/jcmturner/rpc/v2 v2.0.3 // indirect
 	github.com/jonboulle/clockwork v0.2.2 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
-	github.com/klauspost/cpuid/v2 v2.1.1 // indirect
+	github.com/klauspost/cpuid/v2 v2.2.3 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-runewidth v0.0.14 // indirect
 	github.com/mattn/go-sqlite3 v2.0.1+incompatible // indirect
@@ -228,11 +230,11 @@ require (
 	go.etcd.io/etcd/api/v3 v3.5.7 // indirect
 	go.etcd.io/etcd/client/pkg/v3 v3.5.7 // indirect
 	go.uber.org/atomic v1.10.0 // indirect
-	go.uber.org/multierr v1.8.0 // indirect
+	go.uber.org/multierr v1.9.0 // indirect
 	go.uber.org/zap v1.24.0 // indirect
 	golang.org/x/mod v0.8.0 // indirect
 	golang.org/x/sync v0.1.0 // indirect
-	golang.org/x/term v0.5.0 // indirect
+	golang.org/x/term v0.6.0 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect

k8s/charts/seaweedfs/templates/filer-statefulset.yaml

@@ -97,6 +97,12 @@ spec:
               value: {{ $value | quote }}
             {{- end }}
             {{- end }}
+            {{- if .Values.filer.secretExtraEnvironmentVars }}
+            {{- range $key, $value := .Values.filer.secretExtraEnvironmentVars }}
+            - name: {{ $key }}
+              valueFrom: {{ toYaml $value | nindent 16 }}
+            {{- end }}
+            {{- end }}             
           command:
             - "/bin/sh"
             - "-ec"

k8s/charts/seaweedfs/values.yaml

@@ -407,6 +407,18 @@ filer:
     # directories under this folder will be automatically creating a separate bucket
     WEED_FILER_BUCKETS_FOLDER: "/buckets"
 
+
+  # secret env variables
+  secretExtraEnvironmentVars: []
+      # WEED_POSTGRES_USERNAME:
+      #   secretKeyRef:
+      #     name: postgres-credentials
+      #     key: username
+      # WEED_POSTGRES_PASSWORD:
+      #   secretKeyRef:
+      #     name: postgres-credentials
+      #     key: password
+
   s3:
     enabled: true
     port: 8333

weed/command/master.go

@@ -2,13 +2,14 @@ package command
 
 import (
 	"fmt"
-	hashicorpRaft "github.com/hashicorp/raft"
 	"net/http"
 	"os"
 	"path"
 	"strings"
 	"time"
 
+	hashicorpRaft "github.com/hashicorp/raft"
+
 	"golang.org/x/exp/slices"
 
 	"github.com/gorilla/mux"
@@ -163,11 +164,11 @@ func startMaster(masterOption MasterOptions, masterWhiteList []string) {
 		RaftResumeState:   *masterOption.raftResumeState,
 		HeartbeatInterval: *masterOption.heartbeatInterval,
 		ElectionTimeout:   *masterOption.electionTimeout,
-		RaftBootstrap:     *m.raftBootstrap,
+		RaftBootstrap:     *masterOption.raftBootstrap,
 	}
 	var raftServer *weed_server.RaftServer
 	var err error
-	if *m.raftHashicorp {
+	if *masterOption.raftHashicorp {
 		if raftServer, err = weed_server.NewHashicorpRaftServer(raftServerOption); err != nil {
 			glog.Fatalf("NewHashicorpRaftServer: %s", err)
 		}
@@ -180,7 +181,7 @@ func startMaster(masterOption MasterOptions, masterWhiteList []string) {
 	ms.SetRaftServer(raftServer)
 	r.HandleFunc("/cluster/status", raftServer.StatusHandler).Methods("GET")
 	r.HandleFunc("/cluster/healthz", raftServer.HealthzHandler).Methods("GET", "HEAD")
-	if *m.raftHashicorp {
+	if *masterOption.raftHashicorp {
 		r.HandleFunc("/raft/stats", raftServer.StatsRaftHandler).Methods("GET")
 	}
 	// starting grpc server
@@ -191,7 +192,7 @@ func startMaster(masterOption MasterOptions, masterWhiteList []string) {
 	}
 	grpcS := pb.NewGrpcServer(security.LoadServerTLS(util.GetViper(), "grpc.master"))
 	master_pb.RegisterSeaweedServer(grpcS, ms)
-	if *m.raftHashicorp {
+	if *masterOption.raftHashicorp {
 		raftServer.TransportManager.Register(grpcS)
 	} else {
 		protobuf.RegisterRaftServer(grpcS, raftServer)
@@ -204,7 +205,7 @@ func startMaster(masterOption MasterOptions, masterWhiteList []string) {
 	go grpcS.Serve(grpcL)
 
 	timeSleep := 1500 * time.Millisecond
-	if !*m.raftHashicorp {
+	if !*masterOption.raftHashicorp {
 		go func() {
 			time.Sleep(timeSleep)
 

weed/command/server.go

@@ -97,6 +97,7 @@ func init() {
 	masterOptions.metricsIntervalSec = cmdServer.Flag.Int("master.metrics.intervalSeconds", 15, "Prometheus push interval in seconds")
 	masterOptions.raftResumeState = cmdServer.Flag.Bool("master.resumeState", false, "resume previous state on start master server")
 	masterOptions.raftHashicorp = cmdServer.Flag.Bool("master.raftHashicorp", false, "use hashicorp raft")
+	masterOptions.raftBootstrap = cmdMaster.Flag.Bool("master.raftBootstrap", false, "Whether to bootstrap the Raft cluster")
 	masterOptions.heartbeatInterval = cmdServer.Flag.Duration("master.heartbeatInterval", 300*time.Millisecond, "heartbeat interval of master servers, and will be randomly multiplied by [1, 1.25)")
 	masterOptions.electionTimeout = cmdServer.Flag.Duration("master.electionTimeout", 10*time.Second, "election timeout of master servers")
 

weed/images/cropping.go

@@ -1,47 +1,47 @@
-package images
-
-import (
-	"bytes"
-	"image"
-	"image/gif"
-	"image/jpeg"
-	"image/png"
-	"io"
-
-	"github.com/disintegration/imaging"
-
-	"github.com/seaweedfs/seaweedfs/weed/glog"
-)
-
-func Cropped(ext string, read io.ReadSeeker, x1, y1, x2, y2 int) (cropped io.ReadSeeker, err error) {
-	srcImage, _, err := image.Decode(read)
-	if err != nil {
-		glog.Error(err)
-		return read, err
-	}
-
-	bounds := srcImage.Bounds()
-	if x2 > bounds.Dx() || y2 > bounds.Dy() {
-		read.Seek(0, 0)
-		return read, nil
-	}
-
-	rectangle := image.Rect(x1, y1, x2, y2)
-	dstImage := imaging.Crop(srcImage, rectangle)
-	var buf bytes.Buffer
-	switch ext {
-	case ".jpg", ".jpeg":
-		if err = jpeg.Encode(&buf, dstImage, nil); err != nil {
-			glog.Error(err)
-		}
-	case ".png":
-		if err = png.Encode(&buf, dstImage); err != nil {
-			glog.Error(err)
-		}
-	case ".gif":
-		if err = gif.Encode(&buf, dstImage, nil); err != nil {
-			glog.Error(err)
-		}
-	}
-	return bytes.NewReader(buf.Bytes()), err
-}
+package images
+
+import (
+	"bytes"
+	"image"
+	"image/gif"
+	"image/jpeg"
+	"image/png"
+	"io"
+
+	"github.com/disintegration/imaging"
+
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+)
+
+func Cropped(ext string, read io.ReadSeeker, x1, y1, x2, y2 int) (cropped io.ReadSeeker, err error) {
+	srcImage, _, err := image.Decode(read)
+	if err != nil {
+		glog.Error(err)
+		return read, err
+	}
+
+	bounds := srcImage.Bounds()
+	if x2 > bounds.Dx() || y2 > bounds.Dy() {
+		read.Seek(0, 0)
+		return read, nil
+	}
+
+	rectangle := image.Rect(x1, y1, x2, y2)
+	dstImage := imaging.Crop(srcImage, rectangle)
+	var buf bytes.Buffer
+	switch ext {
+	case ".jpg", ".jpeg":
+		if err = jpeg.Encode(&buf, dstImage, nil); err != nil {
+			glog.Error(err)
+		}
+	case ".png":
+		if err = png.Encode(&buf, dstImage); err != nil {
+			glog.Error(err)
+		}
+	case ".gif":
+		if err = gif.Encode(&buf, dstImage, nil); err != nil {
+			glog.Error(err)
+		}
+	}
+	return bytes.NewReader(buf.Bytes()), err
+}

weed/mount/weedfs_dir_mkrm.go

@@ -3,14 +3,16 @@ package mount
 import (
 	"context"
 	"fmt"
-	"github.com/hanwen/go-fuse/v2/fuse"
-	"github.com/seaweedfs/seaweedfs/weed/filer"
-	"github.com/seaweedfs/seaweedfs/weed/glog"
-	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
 	"os"
 	"strings"
 	"syscall"
 	"time"
+
+	"github.com/hanwen/go-fuse/v2/fuse"
+
+	"github.com/seaweedfs/seaweedfs/weed/filer"
+	"github.com/seaweedfs/seaweedfs/weed/glog"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
 )
 
 /** Create a directory
@@ -54,9 +56,10 @@ func (wfs *WFS) Mkdir(cancel <-chan struct{}, in *fuse.MkdirIn, name string, out
 		defer wfs.mapPbIdFromFilerToLocal(newEntry)
 
 		request := &filer_pb.CreateEntryRequest{
-			Directory:  string(dirFullPath),
-			Entry:      newEntry,
-			Signatures: []int32{wfs.signature},
+			Directory:                string(dirFullPath),
+			Entry:                    newEntry,
+			Signatures:               []int32{wfs.signature},
+			SkipCheckParentDirectory: true,
 		}
 
 		glog.V(1).Infof("mkdir: %v", request)

weed/mount/weedfs_link.go

@@ -2,12 +2,14 @@ package mount
 
 import (
 	"context"
+	"syscall"
+	"time"
+
 	"github.com/hanwen/go-fuse/v2/fuse"
+
 	"github.com/seaweedfs/seaweedfs/weed/filer"
 	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
-	"syscall"
-	"time"
 )
 
 /*
@@ -72,7 +74,8 @@ func (wfs *WFS) Link(cancel <-chan struct{}, in *fuse.LinkIn, name string, out *
 			HardLinkId:      oldEntry.HardLinkId,
 			HardLinkCounter: oldEntry.HardLinkCounter,
 		},
-		Signatures: []int32{wfs.signature},
+		Signatures:               []int32{wfs.signature},
+		SkipCheckParentDirectory: true,
 	}
 
 	// apply changes to the filer, and also apply to local metaCache

weed/mount/weedfs_symlink.go

@@ -3,13 +3,15 @@ package mount
 import (
 	"context"
 	"fmt"
+	"os"
+	"syscall"
+	"time"
+
 	"github.com/hanwen/go-fuse/v2/fuse"
+
 	"github.com/seaweedfs/seaweedfs/weed/filer"
 	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
-	"os"
-	"syscall"
-	"time"
 )
 
 /** Create a symbolic link */
@@ -42,7 +44,8 @@ func (wfs *WFS) Symlink(cancel <-chan struct{}, header *fuse.InHeader, target st
 				SymlinkTarget: target,
 			},
 		},
-		Signatures: []int32{wfs.signature},
+		Signatures:               []int32{wfs.signature},
+		SkipCheckParentDirectory: true,
 	}
 
 	err := wfs.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {

weed/s3api/s3_constants/acp_grantee_group.go

@@ -1,6 +1,6 @@
 package s3_constants
 
-//Amazon S3 predefined groups
+// Amazon S3 predefined groups
 var (
 	GranteeGroupAllUsers           = "http://acs.amazonaws.com/groups/global/AllUsers"
 	GranteeGroupAuthenticatedUsers = "http://acs.amazonaws.com/groups/global/AuthenticatedUsers"

weed/server/filer_grpc_server_remote.go

@@ -154,10 +154,11 @@ func (fs *FilerServer) CacheRemoteObjectToLocalCluster(ctx context.Context, req
 			}
 
 			chunks = append(chunks, &filer_pb.FileChunk{
+
 				FileId:       assignResult.Fid,
 				Offset:       localOffset,
 				Size:         uint64(size),
-				ModifiedTsNs: time.Now().Unix(),
+				ModifiedTsNs: time.Now().UnixNano(),
 				ETag:         etag,
 				Fid: &filer_pb.FileId{
 					VolumeId: uint32(fileId.VolumeId),

weed/server/filer_server.go

@@ -195,9 +195,6 @@ func (fs *FilerServer) checkWithMaster() {
 					return fmt.Errorf("get master %s configuration: %v", master, err)
 				}
 				fs.metricsAddress, fs.metricsIntervalSec = resp.MetricsAddress, int(resp.MetricsIntervalSeconds)
-				if fs.option.DefaultReplication == "" {
-					fs.option.DefaultReplication = resp.DefaultReplication
-				}
 				return nil
 			})
 			if readErr == nil {

weed/server/filer_server_handlers_read_dir.go

@@ -27,7 +27,7 @@ func (fs *FilerServer) listDirectoryHandler(w http.ResponseWriter, r *http.Reque
 
 	limit, limit_err := strconv.Atoi(r.FormValue("limit"))
 	if limit_err != nil {
-		limit = 100
+		limit = fs.option.DirListingLimit
 	}
 
 	lastFileName := r.FormValue("lastFileName")

weed/server/filer_server_handlers_write_autochunk.go

@@ -49,7 +49,7 @@ func (fs *FilerServer) autoChunk(ctx context.Context, w http.ResponseWriter, r *
 	if err != nil {
 		if strings.HasPrefix(err.Error(), "read input:") || err.Error() == io.ErrUnexpectedEOF.Error() {
 			writeJsonError(w, r, 499, err)
-		} else if strings.HasSuffix(err.Error(), "is a file") {
+		} else if strings.HasSuffix(err.Error(), "is a file") || strings.HasSuffix(err.Error(), "already exists") {
 			writeJsonError(w, r, http.StatusConflict, err)
 		} else {
 			writeJsonError(w, r, http.StatusInternalServerError, err)

weed/shell/command_s3_bucket_list.go

@@ -63,7 +63,7 @@ func (c *commandS3BucketList) Do(args []string, commandEnv *CommandEnv, writer i
 			collectionSize = collectionInfo.Size
 			fileCount = collectionInfo.FileCount - collectionInfo.DeleteCount
 		}
-		fmt.Fprintf(writer, "  %s\tsize:%.0f\tfile:%.0f", entry.Name, collectionSize, fileCount)
+		fmt.Fprintf(writer, "  %s\tsize:%.0f\tchunk:%.0f", entry.Name, collectionSize, fileCount)
 		if entry.Quota > 0 {
 			fmt.Fprintf(writer, "\tquota:%d\tusage:%.2f%%", entry.Quota, float64(collectionSize)*100/float64(entry.Quota))
 		}

weed/shell/command_volume_fsck.go

@@ -163,6 +163,7 @@ func (c *commandVolumeFsck) Do(args []string, commandEnv *CommandEnv, writer io.
 
 	if *c.findMissingChunksInFiler {
 		// collect all filer file ids and paths
+
 		if err = c.collectFilerFileIdAndPaths(dataNodeVolumeIdToVInfo, *purgeAbsent, collectCutoffFromAtNs); err != nil {
 			return fmt.Errorf("collectFilerFileIdAndPaths: %v", err)
 		}

weed/shell/command_volume_server_evacuate.go

@@ -19,8 +19,8 @@ func init() {
 
 type commandVolumeServerEvacuate struct {
 	topologyInfo *master_pb.TopologyInfo
-	targetServer string
-	volumeRack   string
+	targetServer *string
+	volumeRack   *string
 }
 
 func (c *commandVolumeServerEvacuate) Name() string {
@@ -49,8 +49,8 @@ func (c *commandVolumeServerEvacuate) Do(args []string, commandEnv *CommandEnv,
 
 	vsEvacuateCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
 	volumeServer := vsEvacuateCommand.String("node", "", "<host>:<port> of the volume server")
-	volumeRack := vsEvacuateCommand.String("rack", "", "source rack for the volume servers")
-	targetServer := vsEvacuateCommand.String("target", "", "<host>:<port> of target volume")
+	c.volumeRack = vsEvacuateCommand.String("rack", "", "source rack for the volume servers")
+	c.targetServer = vsEvacuateCommand.String("target", "", "<host>:<port> of target volume")
 	skipNonMoveable := vsEvacuateCommand.Bool("skipNonMoveable", false, "skip volumes that can not be moved")
 	applyChange := vsEvacuateCommand.Bool("force", false, "actually apply the changes")
 	retryCount := vsEvacuateCommand.Int("retry", 0, "how many times to retry")
@@ -63,15 +63,10 @@ func (c *commandVolumeServerEvacuate) Do(args []string, commandEnv *CommandEnv,
 		return
 	}
 
-	if *volumeServer == "" && *volumeRack == "" {
+	if *volumeServer == "" && *c.volumeRack == "" {
 		return fmt.Errorf("need to specify volume server by -node=<host>:<port> or source rack")
 	}
-	if *targetServer != "" {
-		c.targetServer = *targetServer
-	}
-	if *volumeRack != "" {
-		c.volumeRack = *volumeRack
-	}
+
 	for i := 0; i < *retryCount+1; i++ {
 		if err = c.volumeServerEvacuate(commandEnv, *volumeServer, *skipNonMoveable, *applyChange, writer); err == nil {
 			return nil
@@ -237,14 +232,14 @@ func moveAwayOneNormalVolume(commandEnv *CommandEnv, volumeReplicas map[uint32][
 
 func (c *commandVolumeServerEvacuate) nodesOtherThan(volumeServers []*Node, thisServer string) (thisNodes []*Node, otherNodes []*Node) {
 	for _, node := range volumeServers {
-		if node.info.Id == thisServer || (c.volumeRack != "" && node.rack == c.volumeRack) {
+		if node.info.Id == thisServer || (*c.volumeRack != "" && node.rack == *c.volumeRack) {
 			thisNodes = append(thisNodes, node)
 			continue
 		}
-		if c.volumeRack != "" && c.volumeRack == node.rack {
+		if *c.volumeRack != "" && *c.volumeRack == node.rack {
 			continue
 		}
-		if c.targetServer != "" && c.targetServer != node.info.Id {
+		if *c.targetServer != "" && *c.targetServer != node.info.Id {
 			continue
 		}
 		otherNodes = append(otherNodes, node)
@@ -254,14 +249,14 @@ func (c *commandVolumeServerEvacuate) nodesOtherThan(volumeServers []*Node, this
 
 func (c *commandVolumeServerEvacuate) ecNodesOtherThan(volumeServers []*EcNode, thisServer string) (thisNodes []*EcNode, otherNodes []*EcNode) {
 	for _, node := range volumeServers {
-		if node.info.Id == thisServer || (c.volumeRack != "" && string(node.rack) == c.volumeRack) {
+		if node.info.Id == thisServer || (*c.volumeRack != "" && string(node.rack) == *c.volumeRack) {
 			thisNodes = append(thisNodes, node)
 			continue
 		}
-		if c.volumeRack != "" && c.volumeRack == string(node.rack) {
+		if *c.volumeRack != "" && *c.volumeRack == string(node.rack) {
 			continue
 		}
-		if c.targetServer != "" && c.targetServer != node.info.Id {
+		if *c.targetServer != "" && *c.targetServer != node.info.Id {
 			continue
 		}
 		otherNodes = append(otherNodes, node)

weed/shell/command_volume_server_evacuate_test.go

@@ -6,7 +6,12 @@ import (
 )
 
 func TestVolumeServerEvacuate(t *testing.T) {
-	c := commandVolumeServerEvacuate{}
+	targetServer := ""
+	volumeRack := "rack1"
+	c := commandVolumeServerEvacuate{
+		targetServer: &targetServer,
+		volumeRack:   &volumeRack,
+	}
 	c.topologyInfo = parseOutput(topoData)
 
 	volumeServer := "192.168.1.4:8080"

weed/stats/metrics.go

@@ -1,7 +1,6 @@
 package stats
 
 import (
-	"fmt"
 	"log"
 	"net"
 	"net/http"
@@ -282,12 +281,21 @@ func LoopPushingMetric(name, instance, addr string, intervalSeconds int) {
 	}
 }
 
+func JoinHostPort(host string, port int) string {
+	portStr := strconv.Itoa(port)
+	if strings.HasPrefix(host, "[") && strings.HasSuffix(host, "]") {
+		return host + ":" + portStr
+	}
+	return net.JoinHostPort(host, portStr)
+}
+
+
 func StartMetricsServer(ip string, port int) {
 	if port == 0 {
 		return
 	}
 	http.Handle("/metrics", promhttp.HandlerFor(Gather, promhttp.HandlerOpts{}))
-	log.Fatal(http.ListenAndServe(fmt.Sprintf("%s:%d", ip, port), nil))
+	log.Fatal(http.ListenAndServe(JoinHostPort(ip, port), nil))
 }
 
 func SourceName(port uint32) string {

weed/storage/needle/needle_read_page.go

@@ -66,8 +66,12 @@ func (n *Needle) ReadNeedleMeta(r backend.BackendStorageFile, offset int64, size
 	if err != nil {
 		return err
 	}
+
 	var index int
-	index, err = n.readNeedleDataVersion2NonData(metaSlice)
+	if size.IsValid() {
+		index, err = n.readNeedleDataVersion2NonData(metaSlice)
+	}
+
 	n.Checksum = CRC(util.BytesToUint32(metaSlice[index : index+NeedleChecksumSize]))
 	if version == Version3 {
 		n.AppendAtNs = util.BytesToUint64(metaSlice[index+NeedleChecksumSize : index+NeedleChecksumSize+TimestampSize])

weed/storage/needle_map_metric_test.go

@@ -16,7 +16,7 @@ func TestFastLoadingNeedleMapMetrics(t *testing.T) {
 
 	for i := 0; i < 10000; i++ {
 		nm.Put(Uint64ToNeedleId(uint64(i+1)), Uint32ToOffset(uint32(0)), Size(1))
-		if rand.Float32() < 0.2 {
+		if rand.Float32() < 0.2 && i > 0 {
 			nm.Delete(Uint64ToNeedleId(uint64(rand.Int63n(int64(i))+1)), Uint32ToOffset(uint32(0)))
 		}
 	}

weed/storage/volume_read_test.go

@@ -41,7 +41,9 @@ func TestReadNeedMetaWithWritesAndUpdates(t *testing.T) {
 		testNeedle.Flags = 0x08
 		v.readNeedleMetaAt(testNeedle, writeInfos[i].offset, writeInfos[i].size)
 		actualLastModifiedTime := testNeedle.LastModified
-		assert.Equal(t, expectedLastUpdateTime, actualLastModifiedTime, "The two words should be the same.")
+		if writeInfos[i].size != 0 {
+			assert.Equal(t, expectedLastUpdateTime, actualLastModifiedTime, "The two words should be the same.")
+		}
 		expectedLastUpdateTime += 2000
 	}
 }
@@ -86,7 +88,9 @@ func TestReadNeedMetaWithDeletesThenWrites(t *testing.T) {
 		testNeedle.Flags = 0x08
 		v.readNeedleMetaAt(testNeedle, writeInfos[i].offset, writeInfos[i].size)
 		actualLastModifiedTime := testNeedle.LastModified
-		assert.Equal(t, expectedLastUpdateTime, actualLastModifiedTime, "The two words should be the same.")
+		if writeInfos[i].size != 0 {
+			assert.Equal(t, expectedLastUpdateTime, actualLastModifiedTime, "The two words should be the same.")
+		}
 		expectedLastUpdateTime += 2000
 	}
 }

weed/topology/node.go

@@ -2,15 +2,16 @@ package topology
 
 import (
 	"errors"
+	"math/rand"
+	"strings"
+	"sync"
+	"sync/atomic"
+
 	"github.com/seaweedfs/seaweedfs/weed/glog"
 	"github.com/seaweedfs/seaweedfs/weed/stats"
 	"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
 	"github.com/seaweedfs/seaweedfs/weed/storage/needle"
 	"github.com/seaweedfs/seaweedfs/weed/storage/types"
-	"math/rand"
-	"strings"
-	"sync"
-	"sync/atomic"
 )
 
 type NodeId string
@@ -73,7 +74,7 @@ func (n *NodeImpl) PickNodesByWeight(numberOfNodes int, option *VolumeGrowOption
 	n.RUnlock()
 	if len(candidates) < numberOfNodes {
 		glog.V(0).Infoln(n.Id(), "failed to pick", numberOfNodes, "from ", len(candidates), "node candidates")
-		return nil, nil, errors.New("No enough data node found!")
+		return nil, nil, errors.New("Not enough data nodes found!")
 	}
 
 	//pick nodes randomly by weights, the node picked earlier has higher final weights

weed/topology/topology.go

@@ -11,6 +11,8 @@ import (
 	"github.com/seaweedfs/seaweedfs/weed/pb"
 	"github.com/seaweedfs/seaweedfs/weed/storage/types"
 
+	backoff "github.com/cenkalti/backoff/v4"
+
 	hashicorpRaft "github.com/hashicorp/raft"
 	"github.com/seaweedfs/raft"
 
@@ -35,7 +37,7 @@ type Topology struct {
 
 	volumeSizeLimit  uint64
 	replicationAsMin bool
-	isDisableVacuum     bool
+	isDisableVacuum  bool
 
 	Sequence sequence.Sequencer
 
@@ -96,19 +98,10 @@ func (t *Topology) IsLeader() bool {
 }
 
 func (t *Topology) Leader() (l pb.ServerAddress, err error) {
-	for count := 0; count < 3; count++ {
-		l, err = t.MaybeLeader()
-		if err != nil {
-			return
-		}
-		if l != "" {
-			break
-		}
-
-		time.Sleep(time.Duration(5+count) * time.Second)
-	}
-
-	return
+	exponentialBackoff := backoff.NewExponentialBackOff()
+	exponentialBackoff.InitialInterval = 100 * time.Millisecond
+	exponentialBackoff.MaxElapsedTime = 20 * time.Second
+	return backoff.RetryWithData(t.MaybeLeader, exponentialBackoff)
 }
 
 func (t *Topology) MaybeLeader() (l pb.ServerAddress, err error) {