Merge branch 'master' of https://github.com/lyang24/seaweedfs into fs_meta_snapshot

.github/workflows/binaries_dev.yml

@@ -38,7 +38,7 @@ jobs:
     steps:
 
       - name: Check out code into the Go module directory
-        uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v2
+        uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # v2
 
       - name: Set BUILD_TIME env
         run: echo BUILD_TIME=$(date -u +%Y%m%d-%H%M) >> ${GITHUB_ENV}
@@ -87,7 +87,7 @@ jobs:
     steps:
 
       - name: Check out code into the Go module directory
-        uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v2
+        uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # v2
 
       - name: Set BUILD_TIME env
         run: echo BUILD_TIME=$(date -u +%Y%m%d-%H%M) >> ${GITHUB_ENV}

.github/workflows/binaries_release0.yml

@@ -28,7 +28,7 @@ jobs:
     # Steps represent a sequence of tasks that will be executed as part of the job
     steps:
       # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
-      - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v2
+      - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # v2
       - name: Go Release Binaries Normal Volume Size
         uses: wangyoucao577/go-release-action@90da8ebfdc010a0e7d378419a76fd90230a05228 # v1.22
         with:

.github/workflows/binaries_release1.yml

@@ -28,7 +28,7 @@ jobs:
     # Steps represent a sequence of tasks that will be executed as part of the job
     steps:
       # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
-      - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v2
+      - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # v2
       - name: Go Release Binaries Normal Volume Size
         uses: wangyoucao577/go-release-action@90da8ebfdc010a0e7d378419a76fd90230a05228 # v1.22
         with:

.github/workflows/binaries_release2.yml

@@ -28,7 +28,7 @@ jobs:
     # Steps represent a sequence of tasks that will be executed as part of the job
     steps:
       # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
-      - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v2
+      - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # v2
       - name: Go Release Binaries Normal Volume Size
         uses: wangyoucao577/go-release-action@90da8ebfdc010a0e7d378419a76fd90230a05228 # v1.22
         with:

.github/workflows/binaries_release3.yml

@@ -28,7 +28,7 @@ jobs:
     # Steps represent a sequence of tasks that will be executed as part of the job
     steps:
       # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
-      - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v2
+      - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # v2
       - name: Go Release Binaries Normal Volume Size
         uses: wangyoucao577/go-release-action@90da8ebfdc010a0e7d378419a76fd90230a05228 # v1.22
         with:

.github/workflows/binaries_release4.yml

@@ -28,7 +28,7 @@ jobs:
     # Steps represent a sequence of tasks that will be executed as part of the job
     steps:
       # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
-      - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v2
+      - uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # v2
       - name: Go Release Binaries Normal Volume Size
         uses: wangyoucao577/go-release-action@90da8ebfdc010a0e7d378419a76fd90230a05228 # v1.22
         with:

.github/workflows/codeql.yml

@@ -18,7 +18,7 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8
+        uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b
 
       # Initializes the CodeQL tools for scanning.
       - name: Initialize CodeQL

.github/workflows/container_dev.yml

@@ -16,7 +16,7 @@ jobs:
     steps:
       -
         name: Checkout
-        uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v2
+        uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # v2
       -
         name: Docker meta
         id: docker_meta

.github/workflows/container_latest.yml

@@ -17,7 +17,7 @@ jobs:
     steps:
       -
         name: Checkout
-        uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v2
+        uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # v2
       -
         name: Docker meta
         id: docker_meta

.github/workflows/container_release1.yml

@@ -16,7 +16,7 @@ jobs:
     steps:
       -
         name: Checkout
-        uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v2
+        uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # v2
       -
         name: Docker meta
         id: docker_meta

.github/workflows/container_release2.yml

@@ -17,7 +17,7 @@ jobs:
     steps:
       -
         name: Checkout
-        uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v2
+        uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # v2
       -
         name: Docker meta
         id: docker_meta

.github/workflows/container_release3.yml

@@ -17,7 +17,7 @@ jobs:
     steps:
       -
         name: Checkout
-        uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v2
+        uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # v2
       -
         name: Docker meta
         id: docker_meta

.github/workflows/container_release4.yml

@@ -16,7 +16,7 @@ jobs:
     steps:
       -
         name: Checkout
-        uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v2
+        uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # v2
       -
         name: Docker meta
         id: docker_meta

.github/workflows/container_release5.yml

@@ -16,7 +16,7 @@ jobs:
     steps:
       -
         name: Checkout
-        uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v2
+        uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # v2
       -
         name: Docker meta
         id: docker_meta

.github/workflows/depsreview.yml

@@ -9,6 +9,6 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: 'Checkout Repository'
-        uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8
+        uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b
       - name: 'Dependency Review'
-        uses: actions/dependency-review-action@11310527b429536e263dc6cc47873e608189ba21
+        uses: actions/dependency-review-action@0ff3da6f81b812d4ec3cf37a04e2308c7a723730

.github/workflows/e2e.yml

@@ -24,13 +24,13 @@ jobs:
     timeout-minutes: 15
     steps:
     - name: Set up Go 1.x
-      uses: actions/setup-go@d0a58c1c4d2b25278816e339b944508c875f3613 # v2
+      uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # v2
       with:
         go-version: ^1.13
       id: go
 
     - name: Check out code into the Go module directory
-      uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v2
+      uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # v2
 
     - name: Install dependencies
       run: |

.github/workflows/go.yml

@@ -21,13 +21,13 @@ jobs:
     steps:
 
     - name: Set up Go 1.x
-      uses: actions/setup-go@d0a58c1c4d2b25278816e339b944508c875f3613 # v2
+      uses: actions/setup-go@6edd4406fa81c3da01a34fa6f6343087c207a568 # v2
       with:
         go-version: ^1.13
       id: go
 
     - name: Check out code into the Go module directory
-      uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v2
+      uses: actions/checkout@755da8c3cf115ac066823e79a1e1788f8940201b # v2
 
     - name: Get dependencies
       run: |

go.mod

@@ -4,12 +4,12 @@ go 1.19
 
 require (
 	cloud.google.com/go v0.105.0 // indirect
-	cloud.google.com/go/pubsub v1.27.1
+	cloud.google.com/go/pubsub v1.28.0
 	cloud.google.com/go/storage v1.28.1
 	github.com/Azure/azure-pipeline-go v0.2.3
 	github.com/Azure/azure-storage-blob-go v0.15.0
 	github.com/Shopify/sarama v1.37.2
-	github.com/aws/aws-sdk-go v1.44.146
+	github.com/aws/aws-sdk-go v1.44.167
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/bwmarrin/snowflake v0.3.0
 	github.com/cespare/xxhash/v2 v2.1.2 // indirect
@@ -80,7 +80,7 @@ require (
 	github.com/prometheus/client_golang v1.14.0
 	github.com/prometheus/client_model v0.3.0 // indirect
 	github.com/prometheus/common v0.37.0 // indirect
-	github.com/prometheus/procfs v0.8.0
+	github.com/prometheus/procfs v0.9.0
 	github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect
 	github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 // indirect
 	github.com/seaweedfs/goexif v2.0.0+incompatible
@@ -119,9 +119,9 @@ require (
 	golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783 // indirect
 	golang.org/x/sys v0.3.0
 	golang.org/x/text v0.5.0 // indirect
-	golang.org/x/tools v0.3.0
+	golang.org/x/tools v0.4.0
 	golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
-	google.golang.org/api v0.104.0
+	google.golang.org/api v0.105.0
 	google.golang.org/appengine v1.6.7 // indirect
 	google.golang.org/genproto v0.0.0-20221206210731-b1a01be3a5f6 // indirect
 	google.golang.org/grpc v1.51.0
@@ -141,7 +141,7 @@ require (
 
 require (
 	github.com/Jille/raft-grpc-transport v1.3.0
-	github.com/arangodb/go-driver v1.4.0
+	github.com/arangodb/go-driver v1.4.1
 	github.com/armon/go-metrics v0.4.1
 	github.com/fluent/fluent-logger-golang v1.9.0
 	github.com/google/flatbuffers v22.11.23+incompatible
@@ -149,9 +149,10 @@ require (
 	github.com/hashicorp/raft v1.3.11
 	github.com/hashicorp/raft-boltdb/v2 v2.2.2
 	github.com/rabbitmq/amqp091-go v1.5.0
+	github.com/schollz/progressbar/v3 v3.12.2
 	github.com/tikv/client-go/v2 v2.0.3
 	github.com/ydb-platform/ydb-go-sdk-auth-environ v0.1.2
-	github.com/ydb-platform/ydb-go-sdk/v3 v3.40.1
+	github.com/ydb-platform/ydb-go-sdk/v3 v3.41.0
 	golang.org/x/sync v0.1.0
 	google.golang.org/grpc/security/advancedtls v0.0.0-20220622233350-5cdb09fa29c1
 )
@@ -196,8 +197,9 @@ require (
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/klauspost/cpuid/v2 v2.1.1 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
-	github.com/mattn/go-runewidth v0.0.7 // indirect
+	github.com/mattn/go-runewidth v0.0.14 // indirect
 	github.com/mattn/go-sqlite3 v2.0.1+incompatible // indirect
+	github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect
 	github.com/montanaflynn/stats v0.6.6 // indirect
 	github.com/nats-io/nats.go v1.16.0 // indirect
 	github.com/nats-io/nkeys v0.3.0 // indirect
@@ -210,6 +212,7 @@ require (
 	github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00 // indirect
 	github.com/pingcap/kvproto v0.0.0-20221129023506-621ec37aac7a // indirect
 	github.com/pingcap/log v1.1.1-0.20221015072633-39906604fb81 // indirect
+	github.com/rivo/uniseg v0.4.3 // indirect
 	github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/stathat/consistent v1.0.0 // indirect
@@ -218,7 +221,7 @@ require (
 	github.com/tinylib/msgp v1.1.6 // indirect
 	github.com/twmb/murmur3 v1.1.3 // indirect
 	github.com/yandex-cloud/go-genproto v0.0.0-20211115083454-9ca41db5ed9e // indirect
-	github.com/ydb-platform/ydb-go-genproto v0.0.0-20220922065549-66df47a830ba // indirect
+	github.com/ydb-platform/ydb-go-genproto v0.0.0-20221215182650-986f9d10542f // indirect
 	github.com/ydb-platform/ydb-go-yc v0.8.3 // indirect
 	github.com/ydb-platform/ydb-go-yc-metadata v0.5.2 // indirect
 	go.etcd.io/bbolt v1.3.6 // indirect
@@ -228,6 +231,7 @@ require (
 	go.uber.org/multierr v1.8.0 // indirect
 	go.uber.org/zap v1.22.0 // indirect
 	golang.org/x/mod v0.7.0 // indirect
+	golang.org/x/term v0.3.0 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect

go.sum

@@ -73,8 +73,8 @@ cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+
 cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA=
 cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU=
 cloud.google.com/go/pubsub v1.24.0/go.mod h1:rWv09Te1SsRpRGPiWOMDKraMQTJyJps4MkUCoMGUgqw=
-cloud.google.com/go/pubsub v1.27.1 h1:q+J/Nfr6Qx4RQeu3rJcnN48SNC0qzlYzSeqkPq93VHs=
-cloud.google.com/go/pubsub v1.27.1/go.mod h1:hQN39ymbV9geqBnfQq6Xf63yNhUAhv9CZhzp5O6qsW0=
+cloud.google.com/go/pubsub v1.28.0 h1:XzabfdPx/+eNrsVVGLFgeUnQQKPGkMb8klRCeYK52is=
+cloud.google.com/go/pubsub v1.28.0/go.mod h1:vuXFpwaVoIPQMGXqRyUQigu/AX1S3IWugR9xznmcXX8=
 cloud.google.com/go/secretmanager v1.5.0/go.mod h1:5C9kM+RwSpkURNovKySkNvGQLUaOgyoR5W0RUx2SyHQ=
 cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
 cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos=
@@ -211,8 +211,8 @@ github.com/antihax/optional v0.0.0-20180407024304-ca021399b1a6/go.mod h1:V8iCPQY
 github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
 github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
 github.com/apache/thrift v0.13.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
-github.com/arangodb/go-driver v1.4.0 h1:uNCbVYkr5ZP3hIVUP6wqjOVyhMYOL9NDmR762tIeYP0=
-github.com/arangodb/go-driver v1.4.0/go.mod h1:5GAx3XvK72DJPhJgyjZOtYAGc4SpY7rZDb3LyhCvLcQ=
+github.com/arangodb/go-driver v1.4.1 h1:Jg0N7XKxiKwjswmAcMCnefWmt81KJEqybqRAGJDRWlo=
+github.com/arangodb/go-driver v1.4.1/go.mod h1:UTtaxTUMmyPWzKc2dsWWOZzZ3yM6aHWxn/eubGa3YmQ=
 github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e h1:Xg+hGrY2LcQBbxd0ZFdbGSyRKTYMZCfBbw/pMJFOk1g=
 github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e/go.mod h1:mq7Shfa/CaixoDxiyAAc5jZ6CVBAyPaNQCGS7mkj4Ho=
 github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
@@ -239,8 +239,8 @@ github.com/aws/aws-sdk-go v1.43.11/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4
 github.com/aws/aws-sdk-go v1.43.31/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo=
 github.com/aws/aws-sdk-go v1.44.45/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo=
 github.com/aws/aws-sdk-go v1.44.68/go.mod h1:y4AeaBuwd2Lk+GepC1E9v0qOiTws0MIWAX4oIKwKHZo=
-github.com/aws/aws-sdk-go v1.44.146 h1:7YdGgPxDPRJu/yYffzZp/H7yHzQ6AqmuNFZPYraaN8I=
-github.com/aws/aws-sdk-go v1.44.146/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
+github.com/aws/aws-sdk-go v1.44.167 h1:kQmBhGdZkQLU7AiHShSkBJ15zr8agy0QeaxXduvyp2E=
+github.com/aws/aws-sdk-go v1.44.167/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
 github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g=
 github.com/aws/aws-sdk-go-v2 v1.16.8 h1:gOe9UPR98XSf7oEJCcojYg+N2/jCRm4DdeIsP85pIyQ=
 github.com/aws/aws-sdk-go-v2 v1.16.8/go.mod h1:6CpKuLXg2w7If3ABZCl/qZ6rEgwtjZTn4eAf4RcEyuw=
@@ -453,7 +453,6 @@ github.com/containers/ocicrypt v1.1.2/go.mod h1:Dm55fwWm1YZAjYRaJ94z2mfZikIyIN4B
 github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
 github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
 github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
-github.com/coreos/go-iptables v0.4.3/go.mod h1:/mVI274lEDI2ns62jHCDnCyBF9Iwsmekav8Dbxlm1MU=
 github.com/coreos/go-iptables v0.4.5/go.mod h1:/mVI274lEDI2ns62jHCDnCyBF9Iwsmekav8Dbxlm1MU=
 github.com/coreos/go-iptables v0.5.0/go.mod h1:/mVI274lEDI2ns62jHCDnCyBF9Iwsmekav8Dbxlm1MU=
 github.com/coreos/go-iptables v0.6.0/go.mod h1:Qe8Bv2Xik5FyTXwgIbLAnv2sWSBmvWdFETJConOQ//Q=
@@ -490,7 +489,6 @@ github.com/d4l3k/messagediff v1.2.1/go.mod h1:Oozbb1TVXFac9FtSIxHBMnBCq2qeH/2KkE
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/dchest/uniuri v0.0.0-20160212164326-8902c56451e9/go.mod h1:GgB8SF9nRG+GqaDtLcwJZsQFhcogVCJ79j4EdT0c2V4=
 github.com/denisenkom/go-mssqldb v0.12.2/go.mod h1:lnIw1mZukFRZDJYQ0Pb833QS2IaC3l5HkEfra2LJ+sk=
 github.com/dennwc/varint v1.0.0/go.mod h1:hnItb35rvZvJrbTALZtY/iQfDs48JKRG1RPpgziApxA=
 github.com/denverdino/aliyungo v0.0.0-20190125010748-a747050bb1ba/go.mod h1:dV8lFg6daOBZbT6/BDGIz6Y3WFGn8juu6G+CQ6LHtl0=
@@ -1092,6 +1090,7 @@ github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/X
 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
 github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
+github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213/go.mod h1:vNUNkEQ1e29fT/6vq2aBdFsgNPmy8qMdSay1npru+Sw=
 github.com/karlseguin/ccache/v2 v2.0.8 h1:lT38cE//uyf6KcFok0rlgXtGFBWxkI6h/qg4tbFyDnA=
 github.com/karlseguin/ccache/v2 v2.0.8/go.mod h1:2BDThcfQMf/c0jnZowt16eW405XIqZPavt+HoYEtcxQ=
 github.com/karlseguin/expect v1.0.2-0.20190806010014-778a5f0c6003 h1:vJ0Snvo+SLMY72r5J4sEfkuE7AFbixEP2qRbEcum/wA=
@@ -1194,8 +1193,8 @@ github.com/mattn/go-isatty v0.0.16 h1:bq3VjFmv/sOjHtdEhmkEV4x1AJtvUvOJ2PFAZ5+peK
 github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
 github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
 github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
-github.com/mattn/go-runewidth v0.0.7 h1:Ei8KR0497xHyKJPAv59M1dkC+rOZCMBJ+t3fZ+twI54=
-github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
+github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU=
+github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
 github.com/mattn/go-shellwords v1.0.3/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vqg+NOMyg4B2o=
 github.com/mattn/go-shellwords v1.0.6/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vqg+NOMyg4B2o=
 github.com/mattn/go-shellwords v1.0.12/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lLtQsUlTZDWQ8Y=
@@ -1218,6 +1217,8 @@ github.com/minio/highwayhash v1.0.2/go.mod h1:BQskDq+xkJ12lmlUUi7U0M5Swg3EWR+dLT
 github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible/go.mod h1:8AuVvqP/mXw1px98n46wfvcGfQ4ci2FwoAjKYxuo3Z4=
 github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
 github.com/mitchellh/cli v1.1.0/go.mod h1:xcISNoH86gajksDmfB23e/pu+B+GeFRMYmoHXxx3xhI=
+github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ=
+github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw=
 github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
 github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
 github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI=
@@ -1477,8 +1478,8 @@ github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4O
 github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU=
 github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
 github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
-github.com/prometheus/procfs v0.8.0 h1:ODq8ZFEaYeCaZOJlZZdJA2AbQR98dSHSM1KW/You5mo=
-github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4=
+github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJfhI=
+github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY=
 github.com/prometheus/prometheus v0.35.0/go.mod h1:7HaLx5kEPKJ0GDgbODG0fZgXbQ8K/XjZNJXQmbmgQlY=
 github.com/prometheus/prometheus v0.37.0/go.mod h1:egARUgz+K93zwqsVIAneFlLZefyGOON44WyAp4Xqbbk=
 github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
@@ -1491,6 +1492,9 @@ github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 h1:N/ElC8H3+5X
 github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
 github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 h1:OdAsTTz6OkFY5QxjkYwrChwuRruF69c169dPK26NUlk=
 github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
+github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
+github.com/rivo/uniseg v0.4.3 h1:utMvzDsuh3suAEnhH0RdHmoPbU648o6CvXxTx4SBMOw=
+github.com/rivo/uniseg v0.4.3/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
 github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
 github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
 github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
@@ -1502,7 +1506,6 @@ github.com/rs/cors v1.8.2/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU=
 github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
 github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU=
 github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc=
-github.com/rs/zerolog v1.19.0/go.mod h1:IzD0RJ65iWH0w97OQQebJEvTZYvsCUm9WVLWBQrJRjo=
 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd h1:CmH9+J6ZSsIjUK3dcGsnCnO41eRBOnY12zwkn5qVwgc=
 github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd/go.mod h1:hPqNNc0+uJM6H+SuU8sEs5K5IQeKccPqeSjfgcKGgPk=
@@ -1513,6 +1516,8 @@ github.com/safchain/ethtool v0.0.0-20210803160452-9aa261dae9b1/go.mod h1:Z0q5wiB
 github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E=
 github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
 github.com/scaleway/scaleway-sdk-go v1.0.0-beta.9/go.mod h1:fCa7OJZ/9DRTnOKmxvT6pn+LPWUptQAmHF/SBJUGEcg=
+github.com/schollz/progressbar/v3 v3.12.2 h1:yLqqqpQNMxGxHY8uEshRihaHWwa0rf0yb7/Zrpgq2C0=
+github.com/schollz/progressbar/v3 v3.12.2/go.mod h1:HFJYIYQQJX32UJdyoigUl19xoV6aMwZt6iX/C30RWfg=
 github.com/sclevine/agouti v3.0.0+incompatible/go.mod h1:b4WX9W9L1sfQKXeJf1mUTLZKJ48R1S7H23Ji7oFO5Bw=
 github.com/sclevine/spec v1.2.0/go.mod h1:W4J29eT/Kzv7/b9IWLB055Z+qvVC9vt0Arko24q7p+U=
 github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
@@ -1682,13 +1687,13 @@ github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:
 github.com/yandex-cloud/go-genproto v0.0.0-20211115083454-9ca41db5ed9e h1:9LPdmD1vqadsDQUva6t2O9MbnyvoOgo8nFNPaOIH5U8=
 github.com/yandex-cloud/go-genproto v0.0.0-20211115083454-9ca41db5ed9e/go.mod h1:HEUYX/p8966tMUHHT+TsS0hF/Ca/NYwqprC5WXSDMfE=
 github.com/ydb-platform/ydb-go-genproto v0.0.0-20220203104745-929cf9c248bc/go.mod h1:cc138nptTn9eKptCQl/grxP6pBKpo/bnXDiOxuVZtps=
-github.com/ydb-platform/ydb-go-genproto v0.0.0-20220922065549-66df47a830ba h1:htUAISxEY5MfCLmubsf2EMgN+H62cQTNcTseK5F4cJ0=
-github.com/ydb-platform/ydb-go-genproto v0.0.0-20220922065549-66df47a830ba/go.mod h1:Er+FePu1dNUieD+XTMDduGpQuCPssK5Q4BjF+IIXJ3I=
+github.com/ydb-platform/ydb-go-genproto v0.0.0-20221215182650-986f9d10542f h1:BBczNIM1MJHT7XkIUA8pThXWxJvxoBjcWvne3xwe2RI=
+github.com/ydb-platform/ydb-go-genproto v0.0.0-20221215182650-986f9d10542f/go.mod h1:Er+FePu1dNUieD+XTMDduGpQuCPssK5Q4BjF+IIXJ3I=
 github.com/ydb-platform/ydb-go-sdk-auth-environ v0.1.2 h1:EYSI1kulnHb0H0zt3yOw4cRj4ABMSMGwNe43D+fX7e4=
 github.com/ydb-platform/ydb-go-sdk-auth-environ v0.1.2/go.mod h1:Xfjce+VMU9yJVr1lj60yK2fFPWjB4jr/4cp3K7cjzi4=
 github.com/ydb-platform/ydb-go-sdk/v3 v3.25.3/go.mod h1:PFizF/vJsdAgEwjK3DVSBD52kdmRkWfSIS2q2pA+e88=
-github.com/ydb-platform/ydb-go-sdk/v3 v3.40.1 h1:6a/yMcXT+ZpybYsGvEpFSsOdQlbqtArycUqY7U1Yd1w=
-github.com/ydb-platform/ydb-go-sdk/v3 v3.40.1/go.mod h1:hJqWSE2NZ2o2c9geHtRJee+xwiHgEfQX9koBZPLTfHY=
+github.com/ydb-platform/ydb-go-sdk/v3 v3.41.0 h1:2Kfj7I9EtQ8/o6gLB0mbShrZ/tC3/GFyjVE9z5CCLA0=
+github.com/ydb-platform/ydb-go-sdk/v3 v3.41.0/go.mod h1:4bSfSb2PyBtmbFYsaVg96YxK1NjYBA3opn2o6IYvgZ8=
 github.com/ydb-platform/ydb-go-yc v0.8.3 h1:92UUUMsfvtMl6mho8eQ9lbkiPrF3a9CT+RrVRAKNRwo=
 github.com/ydb-platform/ydb-go-yc v0.8.3/go.mod h1:zUolAFGzJ5XG8uwiseTLr9Lapm7L7hdVdZgLSuv9FXE=
 github.com/ydb-platform/ydb-go-yc-metadata v0.5.2 h1:nMtixUijP0Z7iHJNT9fOL+dbmEzZxqU6Xk87ll7hqXg=
@@ -2199,6 +2204,8 @@ golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9sn
 golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/term v0.3.0 h1:qoo4akIqOcDME5bhc/NgxUdovd6BSS2uMsVjB56q1xI=
+golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -2254,7 +2261,6 @@ golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgw
 golang.org/x/tools v0.0.0-20190706070813-72ffa07ba3db/go.mod h1:jcCCGcm9btYwXyDqrUWc6MKQKKGJCWEQ3AfLSRIbEuI=
 golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20190823170909-c4a336ef6a2f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.0.0-20190828213141-aed303cbaa74/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20190907020128-2ca718005c18/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
@@ -2313,8 +2319,8 @@ golang.org/x/tools v0.1.9/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU=
 golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E=
 golang.org/x/tools v0.1.11/go.mod h1:SgwaegtQh8clINPpECJMqnxLv9I09HLqnW3RMqW0CA4=
 golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
-golang.org/x/tools v0.3.0 h1:SrNbZl6ECOS1qFzgTdQfWXZM9XBkiA6tkFrH9YSTPHM=
-golang.org/x/tools v0.3.0/go.mod h1:/rWhSS2+zyEVwoJf8YAX6L2f0ntZ7Kn/mGgAWcipA5k=
+golang.org/x/tools v0.4.0 h1:7mTAgkunk3fr4GAloyyCasadO6h9zSsQZbwvcaIciV4=
+golang.org/x/tools v0.4.0/go.mod h1:UE5sM2OK9E/d67R0ANs2xJizIymRP5gJU295PvKXxjQ=
 golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -2374,8 +2380,8 @@ google.golang.org/api v0.85.0/go.mod h1:AqZf8Ep9uZ2pyTvgL+x0D3Zt0eoT9b5E8fmzfu6F
 google.golang.org/api v0.86.0/go.mod h1:+Sem1dnrKlrXMR/X0bPnMWyluQe4RsNoYfmNLhOIkzw=
 google.golang.org/api v0.90.0/go.mod h1:+Sem1dnrKlrXMR/X0bPnMWyluQe4RsNoYfmNLhOIkzw=
 google.golang.org/api v0.91.0/go.mod h1:+Sem1dnrKlrXMR/X0bPnMWyluQe4RsNoYfmNLhOIkzw=
-google.golang.org/api v0.104.0 h1:KBfmLRqdZEbwQleFlSLnzpQJwhjpmNOk4cKQIBDZ9mg=
-google.golang.org/api v0.104.0/go.mod h1:JCspTXJbBxa5ySXw4UgUqVer7DfVxbvc/CTUFqAED5U=
+google.golang.org/api v0.105.0 h1:t6P9Jj+6XTn4U9I2wycQai6Q/Kz7iOT+QzjJ3G2V4x8=
+google.golang.org/api v0.105.0/go.mod h1:qh7eD5FJks5+BcE+cjBIm6Gz8vioK7EHvnlniqXBnqI=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=

k8s/helm_charts2/templates/filer-statefulset.yaml

@@ -101,7 +101,12 @@ spec:
             - "/bin/sh"
             - "-ec"
             - | 
-              exec /usr/bin/weed -logdir=/logs \
+              exec /usr/bin/weed \
+              {{- if eq .Values.filer.logs.type "hostPath" }}
+              -logdir=/logs \
+              {{- else }}
+              -logtostderr=true \
+              {{- end }}
               {{- if .Values.filer.loggingOverrideLevel }}
               -v={{ .Values.filer.loggingOverrideLevel }} \
               {{- else }}
@@ -225,13 +230,13 @@ spec:
         {{- if eq .Values.filer.logs.type "hostPath" }}
         - name: seaweedfs-filer-log-volume
           hostPath:
-            path: /storage/logs/seaweedfs/filer
+            path: {{ .Values.filer.logs.hostPathPrefix }}/logs/seaweedfs/filer
             type: DirectoryOrCreate
         {{- end }}
         {{- if eq .Values.filer.data.type "hostPath" }}
         - name: data-filer
           hostPath:
-            path: /storage/filer_store
+            path: {{ .Values.filer.data.hostPathPrefix }}/filer_store
             type: DirectoryOrCreate
         {{- end }}
         - name: db-schema-config-volume

k8s/helm_charts2/templates/master-statefulset.yaml

@@ -90,7 +90,12 @@ spec:
             - "/bin/sh"
             - "-ec"
             - | 
-              exec /usr/bin/weed -logdir=/logs \
+              exec /usr/bin/weed \
+              {{- if eq .Values.master.logs.type "hostPath" }}
+              -logdir=/logs \
+              {{- else }}
+              -logtostderr=true \
+              {{- end }}
               {{- if .Values.master.loggingOverrideLevel }}
               -v={{ .Values.master.loggingOverrideLevel }} \
               {{- else }}
@@ -196,13 +201,13 @@ spec:
         {{- if eq .Values.master.logs.type "hostPath" }}
         - name: seaweedfs-master-log-volume
           hostPath:
-            path: /storage/logs/seaweedfs/master
+            path: {{ .Values.master.logs.hostPathPrefix }}/logs/seaweedfs/master
             type: DirectoryOrCreate
         {{- end }}
         {{- if eq .Values.master.data.type "hostPath" }}
         - name: data-{{ .Release.Namespace }}
           hostPath:
-            path: /ssd/seaweed-master/
+            path: {{ .Values.master.data.hostPathPrefix }}/seaweed-master/
             type: DirectoryOrCreate
         {{- end }}
         {{- if .Values.global.enableSecurity }}

k8s/helm_charts2/templates/s3-deployment.yaml

@@ -72,13 +72,19 @@ spec:
             - "/bin/sh"
             - "-ec"
             - | 
-              exec /usr/bin/weed -logdir=/logs \
+              exec /usr/bin/weed \
+               {{- if eq .Values.s3.logs.type "hostPath" }}
+              -logdir=/logs \
+              {{- else }}
+              -logtostderr=true \
+              {{- end }}
               {{- if .Values.s3.loggingOverrideLevel }}
               -v={{ .Values.s3.loggingOverrideLevel }} \
               {{- else }}
               -v={{ .Values.global.loggingLevel }} \
               {{- end }}
               s3 \
+              -ip.bind={{ .Values.s3.bindAddress }} \
               -port={{ .Values.s3.port }} \
               {{- if .Values.s3.metricsPort }}
               -metricsPort {{ .Values.s3.metricsPort }} \
@@ -167,7 +173,7 @@ spec:
         {{- if eq .Values.s3.logs.type "hostPath" }}
         - name: logs
           hostPath:
-            path: /storage/logs/seaweedfs/s3
+            path: {{ .Values.s3.logs.hostPathPrefix }}/logs/seaweedfs/s3
             type: DirectoryOrCreate
         {{- end }}
         {{- if .Values.global.enableSecurity }}

k8s/helm_charts2/templates/volume-statefulset.yaml

@@ -93,7 +93,12 @@ spec:
             - "/bin/sh"
             - "-ec"
             - |
-              exec /usr/bin/weed -logdir=/logs \
+              exec /usr/bin/weed \
+                {{- if eq .Values.volume.logs.type "hostPath" }}
+                -logdir=/logs \
+                {{- else }}
+                -logtostderr=true \
+                {{- end }}
                 {{- if .Values.volume.loggingOverrideLevel }}
                 -v={{ .Values.volume.loggingOverrideLevel }} \
                 {{- else }}
@@ -206,19 +211,19 @@ spec:
         {{- if eq .Values.volume.data.type "hostPath" }}
         - name: data
           hostPath:
-            path: /storage/object_store/
+            path: {{ .Values.volume.data.hostPathPrefix }}/object_store/
             type: DirectoryOrCreate
         {{- end }}
         {{- if and (eq .Values.volume.idx.type "hostPath") .Values.volume.dir_idx }}
         - name: idx
           hostPath:
-            path: /ssd/seaweedfs-volume-idx/
+            path: {{ .Values.volume.idx.hostPathPrefix }}/seaweedfs-volume-idx/
             type: DirectoryOrCreate
         {{- end }}
         {{- if eq .Values.volume.logs.type "hostPath" }}
         - name: logs
           hostPath:
-            path: /storage/logs/seaweedfs/volume
+            path: {{ .Values.volume.logs.hostPathPrefix }}/logs/seaweedfs/volume
             type: DirectoryOrCreate
         {{- end }}
         {{- if .Values.global.enableSecurity }}

k8s/helm_charts2/values.yaml

@@ -68,11 +68,13 @@ master:
     type: "hostPath"
     size: ""
     storageClass: ""
+    hostPathPrefix: /ssd
 
   logs:
     type: "hostPath"
     size: ""
     storageClass: ""
+    hostPathPrefix: /storage
 
   initContainers: ""
 
@@ -176,15 +178,19 @@ volume:
     type: "hostPath"
     size: ""
     storageClass: ""
+    hostPathPrefix: /storage
+
   idx:
     type: "hostPath"
     size: ""
     storageClass: ""
+    hostPathPrefix: /ssd
 
   logs:
     type: "hostPath"
     size: ""
     storageClass: ""
+    hostPathPrefix: /storage
 
   # limit background compaction or copying speed in mega bytes per second
   compactionMBps: "50"
@@ -308,11 +314,13 @@ filer:
     type: "hostPath"
     size: ""
     storageClass: ""
+    hostPathPrefix: /storage
 
   logs:
     type: "hostPath"
     size: ""
     storageClass: ""
+    hostPathPrefix: /storage
 
   initContainers: ""
 
@@ -418,6 +426,7 @@ s3:
   imageTag: null
   restartPolicy: null
   replicas: 1
+  bindAddress: 0.0.0.0
   port: 8333
   metricsPort: 9327
   loggingOverrideLevel: null
@@ -463,7 +472,7 @@ s3:
     type: "hostPath"
     size: ""
     storageClass: ""
-
+    hostPathPrefix: /storage
 
 certificates:
   commonName: "SeaweedFS CA"

weed/command/filer_sync.go

@@ -399,7 +399,11 @@ func genProcessFunction(sourcePath string, targetPath string, excludePaths []str
 				return nil
 			}
 			key := buildKey(dataSink, message, targetPath, sourceNewKey, sourcePath)
-			return dataSink.CreateEntry(key, message.NewEntry, message.Signatures)
+			if err := dataSink.CreateEntry(key, message.NewEntry, message.Signatures); err != nil {
+				return fmt.Errorf("create entry1 : %v", err)
+			} else {
+				return nil
+			}
 		}
 
 		// this is something special?
@@ -427,7 +431,11 @@ func genProcessFunction(sourcePath string, targetPath string, excludePaths []str
 				}
 				// create the new entry
 				newKey := buildKey(dataSink, message, targetPath, sourceNewKey, sourcePath)
-				return dataSink.CreateEntry(newKey, message.NewEntry, message.Signatures)
+				if err := dataSink.CreateEntry(newKey, message.NewEntry, message.Signatures); err != nil {
+					return fmt.Errorf("create entry2 : %v", err)
+				} else {
+					return nil
+				}
 
 			} else {
 				// new key is outside of the watched directory
@@ -441,7 +449,11 @@ func genProcessFunction(sourcePath string, targetPath string, excludePaths []str
 			if strings.HasPrefix(string(sourceNewKey), sourcePath) {
 				// new key is in the watched directory
 				key := buildKey(dataSink, message, targetPath, sourceNewKey, sourcePath)
-				return dataSink.CreateEntry(key, message.NewEntry, message.Signatures)
+				if err := dataSink.CreateEntry(key, message.NewEntry, message.Signatures); err != nil {
+					return fmt.Errorf("create entry3 : %v", err)
+				} else {
+					return nil
+				}
 			} else {
 				// new key is also outside of the watched directory
 				// skip

weed/command/webdav.go

@@ -13,7 +13,7 @@ import (
 	"github.com/seaweedfs/seaweedfs/weed/pb"
 	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
 	"github.com/seaweedfs/seaweedfs/weed/security"
-	"github.com/seaweedfs/seaweedfs/weed/server"
+	weed_server "github.com/seaweedfs/seaweedfs/weed/server"
 	"github.com/seaweedfs/seaweedfs/weed/util"
 )
 
@@ -23,6 +23,7 @@ var (
 
 type WebDavOption struct {
 	filer          *string
+	filerRootPath  *string
 	port           *int
 	collection     *string
 	replication    *string
@@ -44,6 +45,7 @@ func init() {
 	webDavStandaloneOptions.tlsCertificate = cmdWebDav.Flag.String("cert.file", "", "path to the TLS certificate file")
 	webDavStandaloneOptions.cacheDir = cmdWebDav.Flag.String("cacheDir", os.TempDir(), "local cache directory for file chunks")
 	webDavStandaloneOptions.cacheSizeMB = cmdWebDav.Flag.Int64("cacheCapacityMB", 0, "local cache capacity in MB")
+	webDavStandaloneOptions.filerRootPath = cmdWebDav.Flag.String("filer.path", "/", "use this remote path from filer server")
 }
 
 var cmdWebDav = &Command{
@@ -104,6 +106,7 @@ func (wo *WebDavOption) startWebDav() bool {
 
 	ws, webdavServer_err := weed_server.NewWebDavServer(&weed_server.WebDavOption{
 		Filer:          filerAddress,
+		FilerRootPath:  *wo.filerRootPath,
 		GrpcDialOption: grpcDialOption,
 		Collection:     *wo.collection,
 		Replication:    *wo.replication,

weed/filer/filer_notify.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"io"
 	"math"
+	"regexp"
 	"strings"
 	"time"
 
@@ -108,6 +109,10 @@ func (f *Filer) logFlushFunc(startTime, stopTime time.Time, buf []byte) {
 	}
 }
 
+var (
+	VolumeNotFoundPattern = regexp.MustCompile(`volume \d+? not found`)
+)
+
 func (f *Filer) ReadPersistedLogBuffer(startTime time.Time, stopTsNs int64, eachLogEntryFn func(logEntry *filer_pb.LogEntry) error) (lastTsNs int64, isDone bool, err error) {
 
 	startTime = startTime.UTC()
@@ -159,6 +164,10 @@ func (f *Filer) ReadPersistedLogBuffer(startTime time.Time, stopTsNs int64, each
 				if err == io.EOF {
 					continue
 				}
+				if VolumeNotFoundPattern.MatchString(err.Error()) {
+					glog.Warningf("skipping reading %s: %v", hourMinuteEntry.FullPath, err)
+					continue
+				}
 				return lastTsNs, isDone, fmt.Errorf("reading %s: %v", hourMinuteEntry.FullPath, err)
 			}
 			chunkedFileReader.Close()

weed/mount/page_writer/upload_pipeline.go

@@ -6,7 +6,6 @@ import (
 	"github.com/seaweedfs/seaweedfs/weed/util"
 	"sync"
 	"sync/atomic"
-	"time"
 )
 
 type LogicChunkIndex int
@@ -16,16 +15,15 @@ type UploadPipeline struct {
 	uploaderCountCond  *sync.Cond
 	filepath           util.FullPath
 	ChunkSize          int64
-	writableChunks       map[LogicChunkIndex]PageChunk
-	writableChunksLock   sync.Mutex
-	sealedChunks         map[LogicChunkIndex]*SealedChunk
-	sealedChunksLock     sync.Mutex
 	uploaders          *util.LimitedConcurrentExecutor
 	saveToStorageFn    SaveToStorageFunc
-	activeReadChunks     map[LogicChunkIndex]int
-	activeReadChunksLock sync.Mutex
 	writableChunkLimit int
 	swapFile           *SwapFile
+	chunksLock         sync.Mutex
+	writableChunks     map[LogicChunkIndex]PageChunk
+	sealedChunks       map[LogicChunkIndex]*SealedChunk
+	activeReadChunks   map[LogicChunkIndex]int
+	readerCountCond    *sync.Cond
 }
 
 type SealedChunk struct {
@@ -42,7 +40,7 @@ func (sc *SealedChunk) FreeReference(messageOnFree string) {
 }
 
 func NewUploadPipeline(writers *util.LimitedConcurrentExecutor, chunkSize int64, saveToStorageFn SaveToStorageFunc, bufferChunkLimit int, swapFileDir string) *UploadPipeline {
-	return &UploadPipeline{
+	t := &UploadPipeline{
 		ChunkSize:          chunkSize,
 		writableChunks:     make(map[LogicChunkIndex]PageChunk),
 		sealedChunks:       make(map[LogicChunkIndex]*SealedChunk),
@@ -53,11 +51,13 @@ func NewUploadPipeline(writers *util.LimitedConcurrentExecutor, chunkSize int64,
 		writableChunkLimit: bufferChunkLimit,
 		swapFile:           NewSwapFile(swapFileDir, chunkSize),
 	}
+	t.readerCountCond = sync.NewCond(&t.chunksLock)
+	return t
 }
 
 func (up *UploadPipeline) SaveDataAt(p []byte, off int64, isSequential bool) (n int) {
-	up.writableChunksLock.Lock()
-	defer up.writableChunksLock.Unlock()
+	up.chunksLock.Lock()
+	defer up.chunksLock.Unlock()
 
 	logicChunkIndex := LogicChunkIndex(off / up.ChunkSize)
 
@@ -74,7 +74,6 @@ func (up *UploadPipeline) SaveDataAt(p []byte, off int64, isSequential bool) (n
 				}
 			}
 			up.moveToSealed(up.writableChunks[fullestChunkIndex], fullestChunkIndex)
-			delete(up.writableChunks, fullestChunkIndex)
 			// fmt.Printf("flush chunk %d with %d bytes written\n", logicChunkIndex, fullness)
 		}
 		if isSequential &&
@@ -95,13 +94,17 @@ func (up *UploadPipeline) SaveDataAt(p []byte, off int64, isSequential bool) (n
 func (up *UploadPipeline) MaybeReadDataAt(p []byte, off int64) (maxStop int64) {
 	logicChunkIndex := LogicChunkIndex(off / up.ChunkSize)
 
+	up.chunksLock.Lock()
+	defer func() {
+		up.readerCountCond.Signal()
+		up.chunksLock.Unlock()
+	}()
+
 	// read from sealed chunks first
-	up.sealedChunksLock.Lock()
 	sealedChunk, found := up.sealedChunks[logicChunkIndex]
 	if found {
 		sealedChunk.referenceCounter++
 	}
-	up.sealedChunksLock.Unlock()
 	if found {
 		maxStop = sealedChunk.chunk.ReadDataAt(p, off)
 		glog.V(4).Infof("%s read sealed memchunk [%d,%d)", up.filepath, off, maxStop)
@@ -109,8 +112,6 @@ func (up *UploadPipeline) MaybeReadDataAt(p []byte, off int64) (maxStop int64) {
 	}
 
 	// read from writable chunks last
-	up.writableChunksLock.Lock()
-	defer up.writableChunksLock.Unlock()
 	writableChunk, found := up.writableChunks[logicChunkIndex]
 	if !found {
 		return
@@ -123,8 +124,8 @@ func (up *UploadPipeline) MaybeReadDataAt(p []byte, off int64) (maxStop int64) {
 }
 
 func (up *UploadPipeline) FlushAll() {
-	up.writableChunksLock.Lock()
-	defer up.writableChunksLock.Unlock()
+	up.chunksLock.Lock()
+	defer up.chunksLock.Unlock()
 
 	for logicChunkIndex, memChunk := range up.writableChunks {
 		up.moveToSealed(memChunk, logicChunkIndex)
@@ -143,8 +144,6 @@ func (up *UploadPipeline) moveToSealed(memChunk PageChunk, logicChunkIndex Logic
 	atomic.AddInt32(&up.uploaderCount, 1)
 	glog.V(4).Infof("%s uploaderCount %d ++> %d", up.filepath, up.uploaderCount-1, up.uploaderCount)
 
-	up.sealedChunksLock.Lock()
-
 	if oldMemChunk, found := up.sealedChunks[logicChunkIndex]; found {
 		oldMemChunk.FreeReference(fmt.Sprintf("%s replace chunk %d", up.filepath, logicChunkIndex))
 	}
@@ -155,8 +154,8 @@ func (up *UploadPipeline) moveToSealed(memChunk PageChunk, logicChunkIndex Logic
 	up.sealedChunks[logicChunkIndex] = sealedChunk
 	delete(up.writableChunks, logicChunkIndex)
 
-	up.sealedChunksLock.Unlock()
-
+	// unlock before submitting the uploading jobs
+	up.chunksLock.Unlock()
 	up.uploaders.Execute(func() {
 		// first add to the file chunks
 		sealedChunk.chunk.SaveContent(up.saveToStorageFn)
@@ -172,24 +171,25 @@ func (up *UploadPipeline) moveToSealed(memChunk PageChunk, logicChunkIndex Logic
 		up.uploaderCountCond.L.Unlock()
 
 		// wait for readers
+		up.chunksLock.Lock()
+		defer up.chunksLock.Unlock()
 		for up.IsLocked(logicChunkIndex) {
-			time.Sleep(59 * time.Millisecond)
+			up.readerCountCond.Wait()
 		}
 
 		// then remove from sealed chunks
-		up.sealedChunksLock.Lock()
-		defer up.sealedChunksLock.Unlock()
 		delete(up.sealedChunks, logicChunkIndex)
 		sealedChunk.FreeReference(fmt.Sprintf("%s finished uploading chunk %d", up.filepath, logicChunkIndex))
 
 	})
+	up.chunksLock.Lock()
 }
 
 func (up *UploadPipeline) Shutdown() {
 	up.swapFile.FreeResource()
 
-	up.sealedChunksLock.Lock()
-	defer up.sealedChunksLock.Unlock()
+	up.chunksLock.Lock()
+	defer up.chunksLock.Unlock()
 	for logicChunkIndex, sealedChunk := range up.sealedChunks {
 		sealedChunk.FreeReference(fmt.Sprintf("%s uploadpipeline shutdown chunk %d", up.filepath, logicChunkIndex))
 	}

weed/mount/page_writer/upload_pipeline_lock.go

@@ -10,8 +10,8 @@ func (up *UploadPipeline) LockForRead(startOffset, stopOffset int64) {
 	if stopOffset%up.ChunkSize > 0 {
 		stopLogicChunkIndex += 1
 	}
-	up.activeReadChunksLock.Lock()
-	defer up.activeReadChunksLock.Unlock()
+	up.chunksLock.Lock()
+	defer up.chunksLock.Unlock()
 	for i := startLogicChunkIndex; i < stopLogicChunkIndex; i++ {
 		if count, found := up.activeReadChunks[i]; found {
 			up.activeReadChunks[i] = count + 1
@@ -27,8 +27,8 @@ func (up *UploadPipeline) UnlockForRead(startOffset, stopOffset int64) {
 	if stopOffset%up.ChunkSize > 0 {
 		stopLogicChunkIndex += 1
 	}
-	up.activeReadChunksLock.Lock()
-	defer up.activeReadChunksLock.Unlock()
+	up.chunksLock.Lock()
+	defer up.chunksLock.Unlock()
 	for i := startLogicChunkIndex; i < stopLogicChunkIndex; i++ {
 		if count, found := up.activeReadChunks[i]; found {
 			if count == 1 {
@@ -41,8 +41,6 @@ func (up *UploadPipeline) UnlockForRead(startOffset, stopOffset int64) {
 }
 
 func (up *UploadPipeline) IsLocked(logicChunkIndex LogicChunkIndex) bool {
-	up.activeReadChunksLock.Lock()
-	defer up.activeReadChunksLock.Unlock()
 	if count, found := up.activeReadChunks[logicChunkIndex]; found {
 		return count > 0
 	}

weed/replication/sink/filersink/fetch_write.go

@@ -2,9 +2,11 @@ package filersink
 
 import (
 	"fmt"
-	"sync"
-
+	"github.com/schollz/progressbar/v3"
 	"github.com/seaweedfs/seaweedfs/weed/util"
+	"os"
+	"path/filepath"
+	"sync"
 
 	"google.golang.org/grpc"
 
@@ -19,6 +21,20 @@ func (fs *FilerSink) replicateChunks(sourceChunks []*filer_pb.FileChunk, path st
 		return
 	}
 
+	// a simple progress bar. Not ideal. Fix me.
+	var bar *progressbar.ProgressBar
+	if len(sourceChunks) > 1 {
+		name := filepath.Base(path)
+		bar = progressbar.NewOptions64(int64(len(sourceChunks)),
+			progressbar.OptionClearOnFinish(),
+			progressbar.OptionOnCompletion(func() {
+				fmt.Fprint(os.Stderr, "\n")
+			}),
+			progressbar.OptionFullWidth(),
+			progressbar.OptionSetDescription(name),
+		)
+	}
+
 	replicatedChunks = make([]*filer_pb.FileChunk, len(sourceChunks))
 
 	var wg sync.WaitGroup
@@ -27,12 +43,19 @@ func (fs *FilerSink) replicateChunks(sourceChunks []*filer_pb.FileChunk, path st
 		index, source := chunkIndex, sourceChunk
 		fs.executor.Execute(func() {
 			defer wg.Done()
+			util.Retry("replicate chunks", func() error {
 				replicatedChunk, e := fs.replicateOneChunk(source, path)
 				if e != nil {
 					err = e
-				return
+					return e
 				}
 				replicatedChunks[index] = replicatedChunk
+				if bar != nil {
+					bar.Add(1)
+				}
+				err = nil
+				return nil
+			})
 		})
 	}
 	wg.Wait()

weed/replication/sink/filersink/filer_sink.go

@@ -112,7 +112,7 @@ func (fs *FilerSink) CreateEntry(key string, entry *filer_pb.Entry, signatures [
 			Directory: dir,
 			Name:      name,
 		}
-		glog.V(1).Infof("lookup: %v", lookupRequest)
+		// glog.V(1).Infof("lookup: %v", lookupRequest)
 		if resp, err := filer_pb.LookupEntry(client, lookupRequest); err == nil {
 			if filer.ETag(resp.Entry) == filer.ETag(entry) {
 				glog.V(3).Infof("already replicated %s", key)
@@ -125,9 +125,10 @@ func (fs *FilerSink) CreateEntry(key string, entry *filer_pb.Entry, signatures [
 		if err != nil {
 			// only warning here since the source chunk may have been deleted already
 			glog.Warningf("replicate entry chunks %s: %v", key, err)
+			return nil
 		}
 
-		glog.V(4).Infof("replicated %s %+v ===> %+v", key, entry.GetChunks(), replicatedChunks)
+		// glog.V(4).Infof("replicated %s %+v ===> %+v", key, entry.GetChunks(), replicatedChunks)
 
 		request := &filer_pb.CreateEntryRequest{
 			Directory: dir,
@@ -205,7 +206,8 @@ func (fs *FilerSink) UpdateEntry(key string, oldEntry *filer_pb.Entry, newParent
 		// replicate the chunks that are new in the source
 		replicatedChunks, err := fs.replicateChunks(newChunks, key)
 		if err != nil {
-			return true, fmt.Errorf("replicate %s chunks error: %v", key, err)
+			glog.Warningf("replicate entry chunks %s: %v", key, err)
+			return true, nil
 		}
 		existingEntry.Chunks = append(existingEntry.GetChunks(), replicatedChunks...)
 		existingEntry.Attributes = newEntry.Attributes

weed/s3api/filer_multipart_test.go

@@ -57,19 +57,19 @@ func Test_findByPartNumber(t *testing.T) {
 	}
 
 	parts := []CompletedPart{
-		CompletedPart{
+		{
 			ETag:       "xxx",
 			PartNumber: 1,
 		},
-		CompletedPart{
+		{
 			ETag:       "lll",
 			PartNumber: 1,
 		},
-		CompletedPart{
+		{
 			ETag:       "yyy",
 			PartNumber: 3,
 		},
-		CompletedPart{
+		{
 			ETag:       "zzz",
 			PartNumber: 5,
 		},

weed/server/webdav_server.go

@@ -26,6 +26,7 @@ import (
 
 type WebDavOption struct {
 	Filer          pb.ServerAddress
+	FilerRootPath  string
 	DomainName     string
 	BucketsPath    string
 	GrpcDialOption grpc.DialOption
@@ -58,6 +59,11 @@ func NewWebDavServer(option *WebDavOption) (ws *WebDavServer, err error) {
 
 	fs, _ := NewWebDavFileSystem(option)
 
+	// Fix no set filer.path , accessing "/" returns "//"
+	if option.FilerRootPath == "/" {
+		option.FilerRootPath = ""
+	}
+
 	ws = &WebDavServer{
 		option:         option,
 		grpcDialOption: security.LoadClientTLS(util.GetViper(), "grpc.filer"),
@@ -195,7 +201,8 @@ func (fs *WebDavFileSystem) Mkdir(ctx context.Context, fullDirPath string, perm
 }
 
 func (fs *WebDavFileSystem) OpenFile(ctx context.Context, fullFilePath string, flag int, perm os.FileMode) (webdav.File, error) {
-
+	// Add filer.path
+	fullFilePath = fs.option.FilerRootPath + fullFilePath
 	glog.V(2).Infof("WebDavFileSystem.OpenFile %v %x", fullFilePath, flag)
 
 	var err error
@@ -367,7 +374,8 @@ func (fs *WebDavFileSystem) stat(ctx context.Context, fullFilePath string) (os.F
 }
 
 func (fs *WebDavFileSystem) Stat(ctx context.Context, name string) (os.FileInfo, error) {
-
+	// Add filer.path
+	name = fs.option.FilerRootPath + name
 	glog.V(2).Infof("WebDavFileSystem.Stat %v", name)
 
 	return fs.stat(ctx, name)

weed/shell/command_fs_verify.go

@@ -0,0 +1,177 @@
+package shell
+
+import (
+	"context"
+	"flag"
+	"fmt"
+	"github.com/seaweedfs/seaweedfs/weed/filer"
+	"github.com/seaweedfs/seaweedfs/weed/operation"
+	"github.com/seaweedfs/seaweedfs/weed/pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/filer_pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/master_pb"
+	"github.com/seaweedfs/seaweedfs/weed/pb/volume_server_pb"
+	"github.com/seaweedfs/seaweedfs/weed/storage"
+	"github.com/seaweedfs/seaweedfs/weed/util"
+	"io"
+	"math"
+	"strings"
+	"time"
+)
+
+func init() {
+	Commands = append(Commands, &commandFsVerify{})
+}
+
+type commandFsVerify struct {
+	env                *CommandEnv
+	volumeIds          map[uint32][]pb.ServerAddress
+	verbose            *bool
+	modifyTimeAgoAtSec int64
+	writer             io.Writer
+}
+
+func (c *commandFsVerify) Name() string {
+	return "fs.verify"
+}
+
+func (c *commandFsVerify) Help() string {
+	return `recursively verify all files under a directory
+
+	fs.verify [-v] [-modifyTimeAgo 1h] /buckets/dir
+
+`
+}
+
+func (c *commandFsVerify) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {
+	c.env = commandEnv
+	c.writer = writer
+
+	fsVerifyCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
+	c.verbose = fsVerifyCommand.Bool("v", false, "print out each processed files")
+	modifyTimeAgo := fsVerifyCommand.Duration("modifyTimeAgo", 0, "only include files after this modify time to verify")
+
+	if err = fsVerifyCommand.Parse(args); err != nil {
+		return err
+	}
+
+	path, parseErr := commandEnv.parseUrl(findInputDirectory(fsVerifyCommand.Args()))
+	if parseErr != nil {
+		return parseErr
+	}
+
+	c.modifyTimeAgoAtSec = int64(modifyTimeAgo.Seconds())
+
+	if err := c.collectVolumeIds(); err != nil {
+		return parseErr
+	}
+
+	fCount, eConut, terr := c.verifyTraverseBfs(path)
+
+	if terr == nil {
+		fmt.Fprintf(writer, "verified %d files, error %d files \n", fCount, eConut)
+	}
+
+	return terr
+
+}
+
+func (c *commandFsVerify) collectVolumeIds() error {
+	c.volumeIds = make(map[uint32][]pb.ServerAddress)
+	topologyInfo, _, err := collectTopologyInfo(c.env, 0)
+	if err != nil {
+		return err
+	}
+	eachDataNode(topologyInfo, func(dc string, rack RackId, nodeInfo *master_pb.DataNodeInfo) {
+		for _, diskInfo := range nodeInfo.DiskInfos {
+			for _, vi := range diskInfo.VolumeInfos {
+				c.volumeIds[vi.Id] = append(c.volumeIds[vi.Id], pb.NewServerAddressFromDataNode(nodeInfo))
+			}
+		}
+	})
+	return nil
+}
+
+func (c *commandFsVerify) verifyEntry(fileId *filer_pb.FileId, volumeServer *pb.ServerAddress) error {
+	err := operation.WithVolumeServerClient(false, *volumeServer, c.env.option.GrpcDialOption,
+		func(client volume_server_pb.VolumeServerClient) error {
+			_, err := client.VolumeNeedleStatus(context.Background(),
+				&volume_server_pb.VolumeNeedleStatusRequest{
+					VolumeId: fileId.VolumeId,
+					NeedleId: fileId.FileKey})
+			return err
+		},
+	)
+	if err != nil && !strings.Contains(err.Error(), storage.ErrorDeleted.Error()) {
+		return err
+	}
+	if *c.verbose {
+		fmt.Fprintf(c.writer, ".")
+	}
+	return nil
+}
+
+type ItemEntry struct {
+	chunks []*filer_pb.FileChunk
+	path   util.FullPath
+}
+
+func (c *commandFsVerify) verifyTraverseBfs(path string) (fileCount int64, errCount int64, err error) {
+	timeNowAtSec := time.Now().Unix()
+	return fileCount, errCount, doTraverseBfsAndSaving(c.env, nil, path, false,
+		func(entry *filer_pb.FullEntry, outputChan chan interface{}) (err error) {
+			if c.modifyTimeAgoAtSec > 0 {
+				if entry.Entry.Attributes != nil && c.modifyTimeAgoAtSec < timeNowAtSec-entry.Entry.Attributes.Mtime {
+					return nil
+				}
+			}
+			dataChunks, manifestChunks, resolveErr := filer.ResolveChunkManifest(filer.LookupFn(c.env), entry.Entry.GetChunks(), 0, math.MaxInt64)
+			if resolveErr != nil {
+				return fmt.Errorf("failed to ResolveChunkManifest: %+v", resolveErr)
+			}
+			dataChunks = append(dataChunks, manifestChunks...)
+			if len(dataChunks) > 0 {
+				outputChan <- &ItemEntry{
+					chunks: dataChunks,
+					path:   util.NewFullPath(entry.Dir, entry.Entry.Name),
+				}
+			}
+			return nil
+		},
+		func(outputChan chan interface{}) {
+			for itemEntry := range outputChan {
+				i := itemEntry.(*ItemEntry)
+				fileMsg := fmt.Sprintf("file:%s needle status ", i.path)
+				if *c.verbose {
+					fmt.Fprintf(c.writer, fileMsg)
+					fileMsg = ""
+				}
+				for _, chunk := range i.chunks {
+					if volumeIds, ok := c.volumeIds[chunk.Fid.VolumeId]; ok {
+						for _, volumeServer := range volumeIds {
+							if err = c.verifyEntry(chunk.Fid, &volumeServer); err != nil {
+								fmt.Fprintf(c.writer, "%sfailed verify %d:%d: %+v\n",
+									fileMsg, chunk.Fid.VolumeId, chunk.Fid.FileKey, err)
+								break
+							}
+						}
+					} else {
+						err = fmt.Errorf("volumeId %d not found", chunk.Fid.VolumeId)
+						fmt.Fprintf(c.writer, "%sfailed verify chunk %d:%d: %+v\n",
+							fileMsg, chunk.Fid.VolumeId, chunk.Fid.FileKey, err)
+						break
+					}
+				}
+
+				if err != nil {
+					errCount++
+					continue
+				}
+
+				if *c.verbose {
+					fmt.Fprintf(c.writer, " verifed\n")
+				}
+				fileCount++
+			}
+		})
+
+}