Browse Source

Merge branch 'master' into _product

pull/5936/head
changlin.shi 2 years ago
parent
commit
039c950cb2
  1. 2
      .github/workflows/container_dev.yml
  2. 2
      .github/workflows/container_latest.yml
  3. 2
      .github/workflows/container_release1.yml
  4. 2
      .github/workflows/container_release2.yml
  5. 2
      .github/workflows/container_release3.yml
  6. 2
      .github/workflows/container_release4.yml
  7. 2
      .github/workflows/container_release5.yml
  8. 2
      README.md
  9. 84
      go.mod
  10. 361
      go.sum
  11. 6
      k8s/charts/seaweedfs/templates/filer-statefulset.yaml
  12. 12
      k8s/charts/seaweedfs/values.yaml
  13. 13
      weed/command/master.go
  14. 1
      weed/command/server.go
  15. 94
      weed/images/cropping.go
  16. 17
      weed/mount/weedfs_dir_mkrm.go
  17. 9
      weed/mount/weedfs_link.go
  18. 11
      weed/mount/weedfs_symlink.go
  19. 2
      weed/s3api/s3_constants/acp_grantee_group.go
  20. 3
      weed/server/filer_grpc_server_remote.go
  21. 3
      weed/server/filer_server.go
  22. 2
      weed/server/filer_server_handlers_read_dir.go
  23. 2
      weed/server/filer_server_handlers_write_autochunk.go
  24. 2
      weed/shell/command_s3_bucket_list.go
  25. 1
      weed/shell/command_volume_fsck.go
  26. 29
      weed/shell/command_volume_server_evacuate.go
  27. 7
      weed/shell/command_volume_server_evacuate_test.go
  28. 12
      weed/stats/metrics.go
  29. 6
      weed/storage/needle/needle_read_page.go
  30. 2
      weed/storage/needle_map_metric_test.go
  31. 8
      weed/storage/volume_read_test.go
  32. 11
      weed/topology/node.go
  33. 21
      weed/topology/topology.go

2
.github/workflows/container_dev.yml

@ -36,7 +36,7 @@ jobs:
uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 # v1
-
name: Set up Docker Buildx
uses: docker/setup-buildx-action@f03ac48505955848960e80bbb68046aa35c7b9e7 # v1
uses: docker/setup-buildx-action@4b4e9c3e2d4531116a6f8ba8e71fc6e2cb6e6c8c # v1
with:
buildkitd-flags: "--debug"
-

2
.github/workflows/container_latest.yml

@ -37,7 +37,7 @@ jobs:
uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 # v1
-
name: Set up Docker Buildx
uses: docker/setup-buildx-action@f03ac48505955848960e80bbb68046aa35c7b9e7 # v1
uses: docker/setup-buildx-action@4b4e9c3e2d4531116a6f8ba8e71fc6e2cb6e6c8c # v1
with:
buildkitd-flags: "--debug"
-

2
.github/workflows/container_release1.yml

@ -37,7 +37,7 @@ jobs:
uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 # v1
-
name: Set up Docker Buildx
uses: docker/setup-buildx-action@f03ac48505955848960e80bbb68046aa35c7b9e7 # v1
uses: docker/setup-buildx-action@4b4e9c3e2d4531116a6f8ba8e71fc6e2cb6e6c8c # v1
-
name: Login to Docker Hub
if: github.event_name != 'pull_request'

2
.github/workflows/container_release2.yml

@ -38,7 +38,7 @@ jobs:
uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 # v1
-
name: Set up Docker Buildx
uses: docker/setup-buildx-action@f03ac48505955848960e80bbb68046aa35c7b9e7 # v1
uses: docker/setup-buildx-action@4b4e9c3e2d4531116a6f8ba8e71fc6e2cb6e6c8c # v1
-
name: Login to Docker Hub
if: github.event_name != 'pull_request'

2
.github/workflows/container_release3.yml

@ -38,7 +38,7 @@ jobs:
uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 # v1
-
name: Set up Docker Buildx
uses: docker/setup-buildx-action@f03ac48505955848960e80bbb68046aa35c7b9e7 # v1
uses: docker/setup-buildx-action@4b4e9c3e2d4531116a6f8ba8e71fc6e2cb6e6c8c # v1
-
name: Login to Docker Hub
if: github.event_name != 'pull_request'

2
.github/workflows/container_release4.yml

@ -37,7 +37,7 @@ jobs:
uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 # v1
-
name: Set up Docker Buildx
uses: docker/setup-buildx-action@f03ac48505955848960e80bbb68046aa35c7b9e7 # v1
uses: docker/setup-buildx-action@4b4e9c3e2d4531116a6f8ba8e71fc6e2cb6e6c8c # v1
-
name: Login to Docker Hub
if: github.event_name != 'pull_request'

2
.github/workflows/container_release5.yml

@ -37,7 +37,7 @@ jobs:
uses: docker/setup-qemu-action@e81a89b1732b9c48d79cd809d8d81d79c4647a18 # v1
-
name: Set up Docker Buildx
uses: docker/setup-buildx-action@f03ac48505955848960e80bbb68046aa35c7b9e7 # v1
uses: docker/setup-buildx-action@4b4e9c3e2d4531116a6f8ba8e71fc6e2cb6e6c8c # v1
-
name: Login to Docker Hub
if: github.event_name != 'pull_request'

2
README.md

@ -32,7 +32,7 @@ Your support will be really appreciated by me and other supporters!
-->
### Gold Sponsors
[![nodion](https://www.nodion.com/img/logo.svg)](https://www.nodion.com)
[![nodion](https://raw.githubusercontent.com/seaweedfs/seaweedfs/master/note/sponsor_nodion.png)](https://www.nodion.com)
[![piknik](https://raw.githubusercontent.com/seaweedfs/seaweedfs/master/note/piknik.png)](https://www.piknik.com)
---

84
go.mod

@ -3,13 +3,14 @@ module github.com/seaweedfs/seaweedfs
go 1.20
require (
cloud.google.com/go v0.107.0 // indirect
cloud.google.com/go v0.110.0 // indirect
cloud.google.com/go/pubsub v1.28.0
cloud.google.com/go/storage v1.29.0
github.com/Azure/azure-pipeline-go v0.2.3
github.com/Azure/azure-storage-blob-go v0.15.0
github.com/Shopify/sarama v1.38.1
github.com/aws/aws-sdk-go v1.44.209
github.com/aws/aws-sdk-go v1.44.219
github.com/cenkalti/backoff/v4 v4.2.0
github.com/beorn7/perks v1.0.1 // indirect
github.com/bwmarrin/snowflake v0.3.0
github.com/cespare/xxhash/v2 v2.2.0 // indirect
@ -66,7 +67,7 @@ require (
github.com/mailru/easyjson v0.7.7 // indirect
github.com/mattn/go-ieproxy v0.0.9 // indirect
github.com/mattn/go-isatty v0.0.17 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
github.com/mitchellh/mapstructure v1.5.0 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
@ -78,10 +79,10 @@ require (
github.com/pquerna/cachecontrol v0.1.0
github.com/prometheus/client_golang v1.14.0
github.com/prometheus/client_model v0.3.0 // indirect
github.com/prometheus/common v0.37.0 // indirect
github.com/prometheus/common v0.39.0 // indirect
github.com/prometheus/procfs v0.9.0
github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect
github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 // indirect
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
github.com/seaweedfs/goexif v1.0.3
github.com/seaweedfs/raft v1.1.0
github.com/sirupsen/logrus v1.9.0 // indirect
@ -108,39 +109,39 @@ require (
go.etcd.io/etcd/client/v3 v3.5.7
go.mongodb.org/mongo-driver v1.11.1
go.opencensus.io v0.24.0 // indirect
gocloud.dev v0.28.0
gocloud.dev v0.29.0
gocloud.dev/pubsub/natspubsub v0.28.0
gocloud.dev/pubsub/rabbitpubsub v0.28.0
golang.org/x/crypto v0.3.0 // indirect
golang.org/x/exp v0.0.0-20221031165847-c99f073a8326
golang.org/x/image v0.0.0-20200119044424-58c23975cae1
golang.org/x/net v0.7.0
golang.org/x/oauth2 v0.5.0 // indirect
golang.org/x/sys v0.5.0
golang.org/x/text v0.7.0 // indirect
gocloud.dev/pubsub/rabbitpubsub v0.29.0
golang.org/x/crypto v0.6.0 // indirect
golang.org/x/exp v0.0.0-20230124195608-d38c7dcee874
golang.org/x/image v0.6.0
golang.org/x/net v0.8.0
golang.org/x/oauth2 v0.6.0 // indirect
golang.org/x/sys v0.6.0
golang.org/x/text v0.8.0 // indirect
golang.org/x/tools v0.6.0
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
google.golang.org/api v0.110.0
google.golang.org/api v0.112.0
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/genproto v0.0.0-20230209215440-0dfe4f8abfcc // indirect
google.golang.org/genproto v0.0.0-20230303212802-e74f57abe488 // indirect
google.golang.org/grpc v1.53.0
google.golang.org/protobuf v1.28.1
gopkg.in/inf.v0 v0.9.1 // indirect
modernc.org/b v1.0.0 // indirect
modernc.org/cc/v3 v3.40.0 // indirect
modernc.org/ccgo/v3 v3.16.13 // indirect
modernc.org/libc v1.22.2 // indirect
modernc.org/libc v1.22.3 // indirect
modernc.org/mathutil v1.5.0 // indirect
modernc.org/memory v1.4.0 // indirect
modernc.org/memory v1.5.0 // indirect
modernc.org/opt v0.1.3 // indirect
modernc.org/sqlite v1.20.4
modernc.org/sqlite v1.21.0
modernc.org/strutil v1.1.3
modernc.org/token v1.0.1 // indirect
)
require (
github.com/Jille/raft-grpc-transport v1.4.0
github.com/arangodb/go-driver v1.4.1
github.com/arangodb/go-driver v1.5.2
github.com/armon/go-metrics v0.4.1
github.com/fluent/fluent-logger-golang v1.9.0
github.com/google/flatbuffers/go v0.0.0-20230108230133-3b8644d32c50
@ -151,31 +152,32 @@ require (
github.com/schollz/progressbar/v3 v3.13.0
github.com/tikv/client-go/v2 v2.0.5
github.com/ydb-platform/ydb-go-sdk-auth-environ v0.1.2
github.com/ydb-platform/ydb-go-sdk/v3 v3.42.10
github.com/ydb-platform/ydb-go-sdk/v3 v3.43.0
google.golang.org/grpc/security/advancedtls v0.0.0-20220622233350-5cdb09fa29c1
)
require (
cloud.google.com/go/compute v1.18.0 // indirect
cloud.google.com/go/compute/metadata v0.2.3 // indirect
cloud.google.com/go/iam v0.8.0 // indirect
cloud.google.com/go/iam v0.12.0 // indirect
github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e // indirect
github.com/aws/aws-sdk-go-v2 v1.17.1 // indirect
github.com/aws/aws-sdk-go-v2/config v1.18.3 // indirect
github.com/aws/aws-sdk-go-v2/credentials v1.13.3 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.19 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.25 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.19 // indirect
github.com/aws/aws-sdk-go-v2/internal/ini v1.3.26 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.19 // indirect
github.com/aws/aws-sdk-go-v2/service/sns v1.18.6 // indirect
github.com/aws/aws-sdk-go-v2/service/sqs v1.19.15 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.11.25 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.8 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.17.5 // indirect
github.com/aws/smithy-go v1.13.4 // indirect
github.com/benbjohnson/clock v1.1.0 // indirect
github.com/aws/aws-sdk-go-v2 v1.17.4 // indirect
github.com/aws/aws-sdk-go-v2/config v1.18.12 // indirect
github.com/aws/aws-sdk-go-v2/credentials v1.13.12 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.22 // indirect
github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.28 // indirect
github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.22 // indirect
github.com/aws/aws-sdk-go-v2/internal/ini v1.3.29 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.22 // indirect
github.com/aws/aws-sdk-go-v2/service/sns v1.20.2 // indirect
github.com/aws/aws-sdk-go-v2/service/sqs v1.20.2 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.12.1 // indirect
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.14.1 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.18.3 // indirect
github.com/aws/smithy-go v1.13.5 // indirect
github.com/benbjohnson/clock v1.3.0 // indirect
github.com/boltdb/bolt v1.3.1 // indirect
github.com/cenkalti/backoff/v4 v4.2.0
github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 // indirect
github.com/d4l3k/messagediff v1.2.1 // indirect
github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 // indirect
@ -188,13 +190,13 @@ require (
github.com/hashicorp/go-hclog v1.2.0 // indirect
github.com/hashicorp/go-immutable-radix v1.3.1 // indirect
github.com/hashicorp/go-msgpack v1.1.5 // indirect
github.com/hashicorp/golang-lru v0.5.4 // indirect
github.com/hashicorp/golang-lru v0.6.0 // indirect
github.com/jcmturner/aescts/v2 v2.0.0 // indirect
github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect
github.com/jcmturner/rpc/v2 v2.0.3 // indirect
github.com/jonboulle/clockwork v0.2.2 // indirect
github.com/josharian/intern v1.0.0 // indirect
github.com/klauspost/cpuid/v2 v2.1.1 // indirect
github.com/klauspost/cpuid/v2 v2.2.3 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-runewidth v0.0.14 // indirect
github.com/mattn/go-sqlite3 v2.0.1+incompatible // indirect
@ -228,11 +230,11 @@ require (
go.etcd.io/etcd/api/v3 v3.5.7 // indirect
go.etcd.io/etcd/client/pkg/v3 v3.5.7 // indirect
go.uber.org/atomic v1.10.0 // indirect
go.uber.org/multierr v1.8.0 // indirect
go.uber.org/multierr v1.9.0 // indirect
go.uber.org/zap v1.24.0 // indirect
golang.org/x/mod v0.8.0 // indirect
golang.org/x/sync v0.1.0 // indirect
golang.org/x/term v0.5.0 // indirect
golang.org/x/term v0.6.0 // indirect
gopkg.in/ini.v1 v1.67.0 // indirect
gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect

361
go.sum
File diff suppressed because it is too large
View File

6
k8s/charts/seaweedfs/templates/filer-statefulset.yaml

@ -97,6 +97,12 @@ spec:
value: {{ $value | quote }}
{{- end }}
{{- end }}
{{- if .Values.filer.secretExtraEnvironmentVars }}
{{- range $key, $value := .Values.filer.secretExtraEnvironmentVars }}
- name: {{ $key }}
valueFrom: {{ toYaml $value | nindent 16 }}
{{- end }}
{{- end }}
command:
- "/bin/sh"
- "-ec"

12
k8s/charts/seaweedfs/values.yaml

@ -407,6 +407,18 @@ filer:
# directories under this folder will be automatically creating a separate bucket
WEED_FILER_BUCKETS_FOLDER: "/buckets"
# secret env variables
secretExtraEnvironmentVars: []
# WEED_POSTGRES_USERNAME:
# secretKeyRef:
# name: postgres-credentials
# key: username
# WEED_POSTGRES_PASSWORD:
# secretKeyRef:
# name: postgres-credentials
# key: password
s3:
enabled: true
port: 8333

13
weed/command/master.go

@ -2,13 +2,14 @@ package command
import (
"fmt"
hashicorpRaft "github.com/hashicorp/raft"
"net/http"
"os"
"path"
"strings"
"time"
hashicorpRaft "github.com/hashicorp/raft"
"golang.org/x/exp/slices"
"github.com/gorilla/mux"
@ -163,11 +164,11 @@ func startMaster(masterOption MasterOptions, masterWhiteList []string) {
RaftResumeState: *masterOption.raftResumeState,
HeartbeatInterval: *masterOption.heartbeatInterval,
ElectionTimeout: *masterOption.electionTimeout,
RaftBootstrap: *m.raftBootstrap,
RaftBootstrap: *masterOption.raftBootstrap,
}
var raftServer *weed_server.RaftServer
var err error
if *m.raftHashicorp {
if *masterOption.raftHashicorp {
if raftServer, err = weed_server.NewHashicorpRaftServer(raftServerOption); err != nil {
glog.Fatalf("NewHashicorpRaftServer: %s", err)
}
@ -180,7 +181,7 @@ func startMaster(masterOption MasterOptions, masterWhiteList []string) {
ms.SetRaftServer(raftServer)
r.HandleFunc("/cluster/status", raftServer.StatusHandler).Methods("GET")
r.HandleFunc("/cluster/healthz", raftServer.HealthzHandler).Methods("GET", "HEAD")
if *m.raftHashicorp {
if *masterOption.raftHashicorp {
r.HandleFunc("/raft/stats", raftServer.StatsRaftHandler).Methods("GET")
}
// starting grpc server
@ -191,7 +192,7 @@ func startMaster(masterOption MasterOptions, masterWhiteList []string) {
}
grpcS := pb.NewGrpcServer(security.LoadServerTLS(util.GetViper(), "grpc.master"))
master_pb.RegisterSeaweedServer(grpcS, ms)
if *m.raftHashicorp {
if *masterOption.raftHashicorp {
raftServer.TransportManager.Register(grpcS)
} else {
protobuf.RegisterRaftServer(grpcS, raftServer)
@ -204,7 +205,7 @@ func startMaster(masterOption MasterOptions, masterWhiteList []string) {
go grpcS.Serve(grpcL)
timeSleep := 1500 * time.Millisecond
if !*m.raftHashicorp {
if !*masterOption.raftHashicorp {
go func() {
time.Sleep(timeSleep)

1
weed/command/server.go

@ -97,6 +97,7 @@ func init() {
masterOptions.metricsIntervalSec = cmdServer.Flag.Int("master.metrics.intervalSeconds", 15, "Prometheus push interval in seconds")
masterOptions.raftResumeState = cmdServer.Flag.Bool("master.resumeState", false, "resume previous state on start master server")
masterOptions.raftHashicorp = cmdServer.Flag.Bool("master.raftHashicorp", false, "use hashicorp raft")
masterOptions.raftBootstrap = cmdMaster.Flag.Bool("master.raftBootstrap", false, "Whether to bootstrap the Raft cluster")
masterOptions.heartbeatInterval = cmdServer.Flag.Duration("master.heartbeatInterval", 300*time.Millisecond, "heartbeat interval of master servers, and will be randomly multiplied by [1, 1.25)")
masterOptions.electionTimeout = cmdServer.Flag.Duration("master.electionTimeout", 10*time.Second, "election timeout of master servers")

94
weed/images/cropping.go

@ -1,47 +1,47 @@
package images
import (
"bytes"
"image"
"image/gif"
"image/jpeg"
"image/png"
"io"
"github.com/disintegration/imaging"
"github.com/seaweedfs/seaweedfs/weed/glog"
)
func Cropped(ext string, read io.ReadSeeker, x1, y1, x2, y2 int) (cropped io.ReadSeeker, err error) {
srcImage, _, err := image.Decode(read)
if err != nil {
glog.Error(err)
return read, err
}
bounds := srcImage.Bounds()
if x2 > bounds.Dx() || y2 > bounds.Dy() {
read.Seek(0, 0)
return read, nil
}
rectangle := image.Rect(x1, y1, x2, y2)
dstImage := imaging.Crop(srcImage, rectangle)
var buf bytes.Buffer
switch ext {
case ".jpg", ".jpeg":
if err = jpeg.Encode(&buf, dstImage, nil); err != nil {
glog.Error(err)
}
case ".png":
if err = png.Encode(&buf, dstImage); err != nil {
glog.Error(err)
}
case ".gif":
if err = gif.Encode(&buf, dstImage, nil); err != nil {
glog.Error(err)
}
}
return bytes.NewReader(buf.Bytes()), err
}
package images
import (
"bytes"
"image"
"image/gif"
"image/jpeg"
"image/png"
"io"
"github.com/disintegration/imaging"
"github.com/seaweedfs/seaweedfs/weed/glog"
)
func Cropped(ext string, read io.ReadSeeker, x1, y1, x2, y2 int) (cropped io.ReadSeeker, err error) {
srcImage, _, err := image.Decode(read)
if err != nil {
glog.Error(err)
return read, err
}
bounds := srcImage.Bounds()
if x2 > bounds.Dx() || y2 > bounds.Dy() {
read.Seek(0, 0)
return read, nil
}
rectangle := image.Rect(x1, y1, x2, y2)
dstImage := imaging.Crop(srcImage, rectangle)
var buf bytes.Buffer
switch ext {
case ".jpg", ".jpeg":
if err = jpeg.Encode(&buf, dstImage, nil); err != nil {
glog.Error(err)
}
case ".png":
if err = png.Encode(&buf, dstImage); err != nil {
glog.Error(err)
}
case ".gif":
if err = gif.Encode(&buf, dstImage, nil); err != nil {
glog.Error(err)
}
}
return bytes.NewReader(buf.Bytes()), err
}

17
weed/mount/weedfs_dir_mkrm.go

@ -3,14 +3,16 @@ package mount
import (
"context"
"fmt"
"github.com/hanwen/go-fuse/v2/fuse"
"github.com/seaweedfs/seaweedfs/weed/filer"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"os"
"strings"
"syscall"
"time"
"github.com/hanwen/go-fuse/v2/fuse"
"github.com/seaweedfs/seaweedfs/weed/filer"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
)
/** Create a directory
@ -54,9 +56,10 @@ func (wfs *WFS) Mkdir(cancel <-chan struct{}, in *fuse.MkdirIn, name string, out
defer wfs.mapPbIdFromFilerToLocal(newEntry)
request := &filer_pb.CreateEntryRequest{
Directory: string(dirFullPath),
Entry: newEntry,
Signatures: []int32{wfs.signature},
Directory: string(dirFullPath),
Entry: newEntry,
Signatures: []int32{wfs.signature},
SkipCheckParentDirectory: true,
}
glog.V(1).Infof("mkdir: %v", request)

9
weed/mount/weedfs_link.go

@ -2,12 +2,14 @@ package mount
import (
"context"
"syscall"
"time"
"github.com/hanwen/go-fuse/v2/fuse"
"github.com/seaweedfs/seaweedfs/weed/filer"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"syscall"
"time"
)
/*
@ -72,7 +74,8 @@ func (wfs *WFS) Link(cancel <-chan struct{}, in *fuse.LinkIn, name string, out *
HardLinkId: oldEntry.HardLinkId,
HardLinkCounter: oldEntry.HardLinkCounter,
},
Signatures: []int32{wfs.signature},
Signatures: []int32{wfs.signature},
SkipCheckParentDirectory: true,
}
// apply changes to the filer, and also apply to local metaCache

11
weed/mount/weedfs_symlink.go

@ -3,13 +3,15 @@ package mount
import (
"context"
"fmt"
"os"
"syscall"
"time"
"github.com/hanwen/go-fuse/v2/fuse"
"github.com/seaweedfs/seaweedfs/weed/filer"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
"os"
"syscall"
"time"
)
/** Create a symbolic link */
@ -42,7 +44,8 @@ func (wfs *WFS) Symlink(cancel <-chan struct{}, header *fuse.InHeader, target st
SymlinkTarget: target,
},
},
Signatures: []int32{wfs.signature},
Signatures: []int32{wfs.signature},
SkipCheckParentDirectory: true,
}
err := wfs.WithFilerClient(false, func(client filer_pb.SeaweedFilerClient) error {

2
weed/s3api/s3_constants/acp_grantee_group.go

@ -1,6 +1,6 @@
package s3_constants
//Amazon S3 predefined groups
// Amazon S3 predefined groups
var (
GranteeGroupAllUsers = "http://acs.amazonaws.com/groups/global/AllUsers"
GranteeGroupAuthenticatedUsers = "http://acs.amazonaws.com/groups/global/AuthenticatedUsers"

3
weed/server/filer_grpc_server_remote.go

@ -154,10 +154,11 @@ func (fs *FilerServer) CacheRemoteObjectToLocalCluster(ctx context.Context, req
}
chunks = append(chunks, &filer_pb.FileChunk{
FileId: assignResult.Fid,
Offset: localOffset,
Size: uint64(size),
ModifiedTsNs: time.Now().Unix(),
ModifiedTsNs: time.Now().UnixNano(),
ETag: etag,
Fid: &filer_pb.FileId{
VolumeId: uint32(fileId.VolumeId),

3
weed/server/filer_server.go

@ -195,9 +195,6 @@ func (fs *FilerServer) checkWithMaster() {
return fmt.Errorf("get master %s configuration: %v", master, err)
}
fs.metricsAddress, fs.metricsIntervalSec = resp.MetricsAddress, int(resp.MetricsIntervalSeconds)
if fs.option.DefaultReplication == "" {
fs.option.DefaultReplication = resp.DefaultReplication
}
return nil
})
if readErr == nil {

2
weed/server/filer_server_handlers_read_dir.go

@ -27,7 +27,7 @@ func (fs *FilerServer) listDirectoryHandler(w http.ResponseWriter, r *http.Reque
limit, limit_err := strconv.Atoi(r.FormValue("limit"))
if limit_err != nil {
limit = 100
limit = fs.option.DirListingLimit
}
lastFileName := r.FormValue("lastFileName")

2
weed/server/filer_server_handlers_write_autochunk.go

@ -49,7 +49,7 @@ func (fs *FilerServer) autoChunk(ctx context.Context, w http.ResponseWriter, r *
if err != nil {
if strings.HasPrefix(err.Error(), "read input:") || err.Error() == io.ErrUnexpectedEOF.Error() {
writeJsonError(w, r, 499, err)
} else if strings.HasSuffix(err.Error(), "is a file") {
} else if strings.HasSuffix(err.Error(), "is a file") || strings.HasSuffix(err.Error(), "already exists") {
writeJsonError(w, r, http.StatusConflict, err)
} else {
writeJsonError(w, r, http.StatusInternalServerError, err)

2
weed/shell/command_s3_bucket_list.go

@ -63,7 +63,7 @@ func (c *commandS3BucketList) Do(args []string, commandEnv *CommandEnv, writer i
collectionSize = collectionInfo.Size
fileCount = collectionInfo.FileCount - collectionInfo.DeleteCount
}
fmt.Fprintf(writer, " %s\tsize:%.0f\tfile:%.0f", entry.Name, collectionSize, fileCount)
fmt.Fprintf(writer, " %s\tsize:%.0f\tchunk:%.0f", entry.Name, collectionSize, fileCount)
if entry.Quota > 0 {
fmt.Fprintf(writer, "\tquota:%d\tusage:%.2f%%", entry.Quota, float64(collectionSize)*100/float64(entry.Quota))
}

1
weed/shell/command_volume_fsck.go

@ -163,6 +163,7 @@ func (c *commandVolumeFsck) Do(args []string, commandEnv *CommandEnv, writer io.
if *c.findMissingChunksInFiler {
// collect all filer file ids and paths
if err = c.collectFilerFileIdAndPaths(dataNodeVolumeIdToVInfo, *purgeAbsent, collectCutoffFromAtNs); err != nil {
return fmt.Errorf("collectFilerFileIdAndPaths: %v", err)
}

29
weed/shell/command_volume_server_evacuate.go

@ -19,8 +19,8 @@ func init() {
type commandVolumeServerEvacuate struct {
topologyInfo *master_pb.TopologyInfo
targetServer string
volumeRack string
targetServer *string
volumeRack *string
}
func (c *commandVolumeServerEvacuate) Name() string {
@ -49,8 +49,8 @@ func (c *commandVolumeServerEvacuate) Do(args []string, commandEnv *CommandEnv,
vsEvacuateCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
volumeServer := vsEvacuateCommand.String("node", "", "<host>:<port> of the volume server")
volumeRack := vsEvacuateCommand.String("rack", "", "source rack for the volume servers")
targetServer := vsEvacuateCommand.String("target", "", "<host>:<port> of target volume")
c.volumeRack = vsEvacuateCommand.String("rack", "", "source rack for the volume servers")
c.targetServer = vsEvacuateCommand.String("target", "", "<host>:<port> of target volume")
skipNonMoveable := vsEvacuateCommand.Bool("skipNonMoveable", false, "skip volumes that can not be moved")
applyChange := vsEvacuateCommand.Bool("force", false, "actually apply the changes")
retryCount := vsEvacuateCommand.Int("retry", 0, "how many times to retry")
@ -63,15 +63,10 @@ func (c *commandVolumeServerEvacuate) Do(args []string, commandEnv *CommandEnv,
return
}
if *volumeServer == "" && *volumeRack == "" {
if *volumeServer == "" && *c.volumeRack == "" {
return fmt.Errorf("need to specify volume server by -node=<host>:<port> or source rack")
}
if *targetServer != "" {
c.targetServer = *targetServer
}
if *volumeRack != "" {
c.volumeRack = *volumeRack
}
for i := 0; i < *retryCount+1; i++ {
if err = c.volumeServerEvacuate(commandEnv, *volumeServer, *skipNonMoveable, *applyChange, writer); err == nil {
return nil
@ -237,14 +232,14 @@ func moveAwayOneNormalVolume(commandEnv *CommandEnv, volumeReplicas map[uint32][
func (c *commandVolumeServerEvacuate) nodesOtherThan(volumeServers []*Node, thisServer string) (thisNodes []*Node, otherNodes []*Node) {
for _, node := range volumeServers {
if node.info.Id == thisServer || (c.volumeRack != "" && node.rack == c.volumeRack) {
if node.info.Id == thisServer || (*c.volumeRack != "" && node.rack == *c.volumeRack) {
thisNodes = append(thisNodes, node)
continue
}
if c.volumeRack != "" && c.volumeRack == node.rack {
if *c.volumeRack != "" && *c.volumeRack == node.rack {
continue
}
if c.targetServer != "" && c.targetServer != node.info.Id {
if *c.targetServer != "" && *c.targetServer != node.info.Id {
continue
}
otherNodes = append(otherNodes, node)
@ -254,14 +249,14 @@ func (c *commandVolumeServerEvacuate) nodesOtherThan(volumeServers []*Node, this
func (c *commandVolumeServerEvacuate) ecNodesOtherThan(volumeServers []*EcNode, thisServer string) (thisNodes []*EcNode, otherNodes []*EcNode) {
for _, node := range volumeServers {
if node.info.Id == thisServer || (c.volumeRack != "" && string(node.rack) == c.volumeRack) {
if node.info.Id == thisServer || (*c.volumeRack != "" && string(node.rack) == *c.volumeRack) {
thisNodes = append(thisNodes, node)
continue
}
if c.volumeRack != "" && c.volumeRack == string(node.rack) {
if *c.volumeRack != "" && *c.volumeRack == string(node.rack) {
continue
}
if c.targetServer != "" && c.targetServer != node.info.Id {
if *c.targetServer != "" && *c.targetServer != node.info.Id {
continue
}
otherNodes = append(otherNodes, node)

7
weed/shell/command_volume_server_evacuate_test.go

@ -6,7 +6,12 @@ import (
)
func TestVolumeServerEvacuate(t *testing.T) {
c := commandVolumeServerEvacuate{}
targetServer := ""
volumeRack := "rack1"
c := commandVolumeServerEvacuate{
targetServer: &targetServer,
volumeRack: &volumeRack,
}
c.topologyInfo = parseOutput(topoData)
volumeServer := "192.168.1.4:8080"

12
weed/stats/metrics.go

@ -1,7 +1,6 @@
package stats
import (
"fmt"
"log"
"net"
"net/http"
@ -282,12 +281,21 @@ func LoopPushingMetric(name, instance, addr string, intervalSeconds int) {
}
}
func JoinHostPort(host string, port int) string {
portStr := strconv.Itoa(port)
if strings.HasPrefix(host, "[") && strings.HasSuffix(host, "]") {
return host + ":" + portStr
}
return net.JoinHostPort(host, portStr)
}
func StartMetricsServer(ip string, port int) {
if port == 0 {
return
}
http.Handle("/metrics", promhttp.HandlerFor(Gather, promhttp.HandlerOpts{}))
log.Fatal(http.ListenAndServe(fmt.Sprintf("%s:%d", ip, port), nil))
log.Fatal(http.ListenAndServe(JoinHostPort(ip, port), nil))
}
func SourceName(port uint32) string {

6
weed/storage/needle/needle_read_page.go

@ -66,8 +66,12 @@ func (n *Needle) ReadNeedleMeta(r backend.BackendStorageFile, offset int64, size
if err != nil {
return err
}
var index int
index, err = n.readNeedleDataVersion2NonData(metaSlice)
if size.IsValid() {
index, err = n.readNeedleDataVersion2NonData(metaSlice)
}
n.Checksum = CRC(util.BytesToUint32(metaSlice[index : index+NeedleChecksumSize]))
if version == Version3 {
n.AppendAtNs = util.BytesToUint64(metaSlice[index+NeedleChecksumSize : index+NeedleChecksumSize+TimestampSize])

2
weed/storage/needle_map_metric_test.go

@ -16,7 +16,7 @@ func TestFastLoadingNeedleMapMetrics(t *testing.T) {
for i := 0; i < 10000; i++ {
nm.Put(Uint64ToNeedleId(uint64(i+1)), Uint32ToOffset(uint32(0)), Size(1))
if rand.Float32() < 0.2 {
if rand.Float32() < 0.2 && i > 0 {
nm.Delete(Uint64ToNeedleId(uint64(rand.Int63n(int64(i))+1)), Uint32ToOffset(uint32(0)))
}
}

8
weed/storage/volume_read_test.go

@ -41,7 +41,9 @@ func TestReadNeedMetaWithWritesAndUpdates(t *testing.T) {
testNeedle.Flags = 0x08
v.readNeedleMetaAt(testNeedle, writeInfos[i].offset, writeInfos[i].size)
actualLastModifiedTime := testNeedle.LastModified
assert.Equal(t, expectedLastUpdateTime, actualLastModifiedTime, "The two words should be the same.")
if writeInfos[i].size != 0 {
assert.Equal(t, expectedLastUpdateTime, actualLastModifiedTime, "The two words should be the same.")
}
expectedLastUpdateTime += 2000
}
}
@ -86,7 +88,9 @@ func TestReadNeedMetaWithDeletesThenWrites(t *testing.T) {
testNeedle.Flags = 0x08
v.readNeedleMetaAt(testNeedle, writeInfos[i].offset, writeInfos[i].size)
actualLastModifiedTime := testNeedle.LastModified
assert.Equal(t, expectedLastUpdateTime, actualLastModifiedTime, "The two words should be the same.")
if writeInfos[i].size != 0 {
assert.Equal(t, expectedLastUpdateTime, actualLastModifiedTime, "The two words should be the same.")
}
expectedLastUpdateTime += 2000
}
}

11
weed/topology/node.go

@ -2,15 +2,16 @@ package topology
import (
"errors"
"math/rand"
"strings"
"sync"
"sync/atomic"
"github.com/seaweedfs/seaweedfs/weed/glog"
"github.com/seaweedfs/seaweedfs/weed/stats"
"github.com/seaweedfs/seaweedfs/weed/storage/erasure_coding"
"github.com/seaweedfs/seaweedfs/weed/storage/needle"
"github.com/seaweedfs/seaweedfs/weed/storage/types"
"math/rand"
"strings"
"sync"
"sync/atomic"
)
type NodeId string
@ -73,7 +74,7 @@ func (n *NodeImpl) PickNodesByWeight(numberOfNodes int, option *VolumeGrowOption
n.RUnlock()
if len(candidates) < numberOfNodes {
glog.V(0).Infoln(n.Id(), "failed to pick", numberOfNodes, "from ", len(candidates), "node candidates")
return nil, nil, errors.New("No enough data node found!")
return nil, nil, errors.New("Not enough data nodes found!")
}
//pick nodes randomly by weights, the node picked earlier has higher final weights

21
weed/topology/topology.go

@ -11,6 +11,8 @@ import (
"github.com/seaweedfs/seaweedfs/weed/pb"
"github.com/seaweedfs/seaweedfs/weed/storage/types"
backoff "github.com/cenkalti/backoff/v4"
hashicorpRaft "github.com/hashicorp/raft"
"github.com/seaweedfs/raft"
@ -35,7 +37,7 @@ type Topology struct {
volumeSizeLimit uint64
replicationAsMin bool
isDisableVacuum bool
isDisableVacuum bool
Sequence sequence.Sequencer
@ -96,19 +98,10 @@ func (t *Topology) IsLeader() bool {
}
func (t *Topology) Leader() (l pb.ServerAddress, err error) {
for count := 0; count < 3; count++ {
l, err = t.MaybeLeader()
if err != nil {
return
}
if l != "" {
break
}
time.Sleep(time.Duration(5+count) * time.Second)
}
return
exponentialBackoff := backoff.NewExponentialBackOff()
exponentialBackoff.InitialInterval = 100 * time.Millisecond
exponentialBackoff.MaxElapsedTime = 20 * time.Second
return backoff.RetryWithData(t.MaybeLeader, exponentialBackoff)
}
func (t *Topology) MaybeLeader() (l pb.ServerAddress, err error) {

Loading…
Cancel
Save