Merge pull request #82 from chrislusf/master

sync

.github/workflows/binaries_dev.yml

@@ -6,57 +6,66 @@ on:
 
 jobs:
 
+  cleanup:
+    runs-on: ubuntu-latest
 
-  build-latest-docker-image:
-    runs-on: [ubuntu-latest]
+    steps:
+
+      - name: Delete old release assets
+        uses: mknejp/delete-release-assets@v1
+        with:
+          token: ${{ github.token }}
+          tag: dev
+          fail-if-no-assets: false
+          assets: |
+            weed-*
+
+  build_dev:
+    needs: cleanup
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        goos: [linux, windows, darwin, freebsd]
+        goarch: [amd64, arm64]
+        exclude:
+          - goarch: arm64
+            goos: windows
 
     steps:
-      -
-        name: Checkout
+
+      - name: Check out code into the Go module directory
         uses: actions/checkout@v2
-      -
-        name: Docker meta
-        id: docker_meta
-        uses: docker/metadata-action@v3
-        with:
-          images: |
-            chrislusf/seaweedfs
-            ghcr.io/chrislusf/seaweedfs
-          tags: |
-            type=raw,value=latest
-          labels: |
-            org.opencontainers.image.title=seaweedfs
-            org.opencontainers.image.vendor=Chris Lu
-      -
-        name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
-      -
-        name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
-        with:
-          buildkitd-flags: "--debug"
-      -
-        name: Login to Docker Hub
-        if: github.event_name != 'pull_request'
-        uses: docker/login-action@v1
-        with:
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_PASSWORD }}
-      -
-        name: Login to GHCR
-        if: github.event_name != 'pull_request'
-        uses: docker/login-action@v1
+
+      - name: Set BUILD_TIME env
+        run: echo BUILD_TIME=$(date -u +%Y%m%d-%H%M) >> ${GITHUB_ENV}
+
+      - name: Go Release Binaries Large Disk
+        uses: wangyoucao577/go-release-action@v1.20
         with:
-          registry: ghcr.io
-          username: ${{ secrets.GHCR_USERNAME }}
-          password: ${{ secrets.GHCR_TOKEN }}
-      -
-        name: Build
-        uses: docker/build-push-action@v2
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          goos: ${{ matrix.goos }}
+          goarch: ${{ matrix.goarch }}
+          release_tag: dev
+          overwrite: true
+          pre_command: export CGO_ENABLED=0
+          build_flags: -tags 5BytesOffset # optional, default is
+          ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}}
+          # Where to run `go build .`
+          project_path: weed
+          binary_name: weed-large-disk
+          asset_name: "weed-large-disk-${{ env.BUILD_TIME }}-${{ matrix.goos }}-${{ matrix.goarch }}"
+
+      - name: Go Release Binaries Normal Volume Size
+        uses: wangyoucao577/go-release-action@v1.20
         with:
-          context: ./docker
-          push: ${{ github.event_name != 'pull_request' }}
-          file: ./docker/Dockerfile
-          platforms: linux/amd64, linux/arm, linux/arm64
-          tags: ${{ steps.docker_meta.outputs.tags }}
-          labels: ${{ steps.docker_meta.outputs.labels }}
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          goos: ${{ matrix.goos }}
+          goarch: ${{ matrix.goarch }}
+          release_tag: dev
+          overwrite: true
+          pre_command: export CGO_ENABLED=0
+          ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}}
+          # Where to run `go build .`
+          project_path: weed
+          binary_name: weed-normal-disk
+          asset_name: "weed-${{ env.BUILD_TIME }}-${{ matrix.goos }}-${{ matrix.goarch }}"

.github/workflows/binaries_release0.yml

@@ -0,0 +1,54 @@
+# This is a basic workflow to help you get started with Actions
+
+name: "go: build versioned binaries for windows"
+
+on:
+  push:
+    tags:
+      - '*'
+
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+# A workflow run is made up of one or more jobs that can run sequentially or in parallel
+jobs:
+
+  build-release-binaries_windows:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        goos: [windows]
+        goarch: [amd64]
+
+    # Steps represent a sequence of tasks that will be executed as part of the job
+    steps:
+      # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
+      - uses: actions/checkout@v2
+      - name: Go Release Binaries Normal Volume Size
+        uses: wangyoucao577/go-release-action@v1.20
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          goos: ${{ matrix.goos }}
+          goarch: ${{ matrix.goarch }}
+          overwrite: true
+          pre_command: export CGO_ENABLED=0
+          # build_flags: -tags 5BytesOffset # optional, default is
+          ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}}
+          # Where to run `go build .`
+          project_path: weed
+          binary_name: weed
+          asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
+      - name: Go Release Large Disk Binaries
+        uses: wangyoucao577/go-release-action@v1.20
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          goos: ${{ matrix.goos }}
+          goarch: ${{ matrix.goarch }}
+          overwrite: true
+          pre_command: export CGO_ENABLED=0
+          build_flags: -tags 5BytesOffset # optional, default is
+          ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}}
+          # Where to run `go build .`
+          project_path: weed
+          binary_name: weed
+          asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}_large_disk"

.github/workflows/binaries_release.yml → .github/workflows/binaries_release1.yml

@@ -1,6 +1,6 @@
 # This is a basic workflow to help you get started with Actions
 
-name: "go: build versioned binaries"
+name: "go: build versioned binaries for linux"
 
 on:
   push:
@@ -13,21 +13,12 @@ on:
 # A workflow run is made up of one or more jobs that can run sequentially or in parallel
 jobs:
 
-  build-release-binaries:
+  build-release-binaries_linux:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        goos: [linux, windows, darwin, freebsd]
+        goos: [linux]
         goarch: [amd64, arm, arm64]
-        exclude:
-          - goarch: arm
-            goos: darwin
-          - goarch: 386
-            goos: darwin
-          - goarch: arm
-            goos: windows
-          - goarch: arm64
-            goos: windows
 
     # Steps represent a sequence of tasks that will be executed as part of the job
     steps:

.github/workflows/binaries_release2.yml

@@ -0,0 +1,54 @@
+# This is a basic workflow to help you get started with Actions
+
+name: "go: build versioned binaries for darwin"
+
+on:
+  push:
+    tags:
+      - '*'
+
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+# A workflow run is made up of one or more jobs that can run sequentially or in parallel
+jobs:
+
+  build-release-binaries_darwin:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        goos: [darwin]
+        goarch: [amd64, arm64]
+
+    # Steps represent a sequence of tasks that will be executed as part of the job
+    steps:
+      # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
+      - uses: actions/checkout@v2
+      - name: Go Release Binaries Normal Volume Size
+        uses: wangyoucao577/go-release-action@v1.20
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          goos: ${{ matrix.goos }}
+          goarch: ${{ matrix.goarch }}
+          overwrite: true
+          pre_command: export CGO_ENABLED=0
+          # build_flags: -tags 5BytesOffset # optional, default is
+          ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}}
+          # Where to run `go build .`
+          project_path: weed
+          binary_name: weed
+          asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
+      - name: Go Release Large Disk Binaries
+        uses: wangyoucao577/go-release-action@v1.20
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          goos: ${{ matrix.goos }}
+          goarch: ${{ matrix.goarch }}
+          overwrite: true
+          pre_command: export CGO_ENABLED=0
+          build_flags: -tags 5BytesOffset # optional, default is
+          ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}}
+          # Where to run `go build .`
+          project_path: weed
+          binary_name: weed
+          asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}_large_disk"

.github/workflows/binaries_release3.yml

@@ -0,0 +1,54 @@
+# This is a basic workflow to help you get started with Actions
+
+name: "go: build versioned binaries for freebsd"
+
+on:
+  push:
+    tags:
+      - '*'
+
+  # Allows you to run this workflow manually from the Actions tab
+  workflow_dispatch:
+
+# A workflow run is made up of one or more jobs that can run sequentially or in parallel
+jobs:
+
+  build-release-binaries_freebsd:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        goos: [freebsd]
+        goarch: [amd64, arm, arm64]
+
+    # Steps represent a sequence of tasks that will be executed as part of the job
+    steps:
+      # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
+      - uses: actions/checkout@v2
+      - name: Go Release Binaries Normal Volume Size
+        uses: wangyoucao577/go-release-action@v1.20
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          goos: ${{ matrix.goos }}
+          goarch: ${{ matrix.goarch }}
+          overwrite: true
+          pre_command: export CGO_ENABLED=0
+          # build_flags: -tags 5BytesOffset # optional, default is
+          ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}}
+          # Where to run `go build .`
+          project_path: weed
+          binary_name: weed
+          asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
+      - name: Go Release Large Disk Binaries
+        uses: wangyoucao577/go-release-action@v1.20
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          goos: ${{ matrix.goos }}
+          goarch: ${{ matrix.goarch }}
+          overwrite: true
+          pre_command: export CGO_ENABLED=0
+          build_flags: -tags 5BytesOffset # optional, default is
+          ldflags: -extldflags -static -X github.com/chrislusf/seaweedfs/weed/util.COMMIT=${{github.sha}}
+          # Where to run `go build .`
+          project_path: weed
+          binary_name: weed
+          asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}_large_disk"

.github/workflows/container_dev.yml

@@ -58,6 +58,6 @@ jobs:
           context: ./docker
           push: ${{ github.event_name != 'pull_request' }}
           file: ./docker/Dockerfile.go_build
-          platforms: linux/amd64, linux/arm, linux/arm64, linux/386
+          platforms: linux/amd64, linux/arm64
           tags: ${{ steps.docker_meta.outputs.tags }}
           labels: ${{ steps.docker_meta.outputs.labels }}

.github/workflows/container_release.yml

@@ -1,121 +0,0 @@
-name: "docker: build release containers"
-
-on:
-  push:
-    tags:
-      - '*'
-  workflow_dispatch: []
-
-jobs:
-  build-default-release-container:
-    runs-on: [ubuntu-latest]
-
-    steps:
-      -
-        name: Checkout
-        uses: actions/checkout@v2
-      -
-        name: Docker meta
-        id: docker_meta
-        uses: docker/metadata-action@v3
-        with:
-          images: |
-            chrislusf/seaweedfs
-            ghcr.io/chrislusf/seaweedfs
-          tags: |
-            type=ref,event=tag
-          flavor: |
-            latest=false
-          labels: |
-            org.opencontainers.image.title=seaweedfs
-            org.opencontainers.image.description=SeaweedFS is a distributed storage system for blobs, objects, files, and data lake, to store and serve billions of files fast!
-            org.opencontainers.image.vendor=Chris Lu
-      -
-        name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
-      -
-        name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
-        with:
-          buildkitd-flags: "--debug"
-      -
-        name: Login to Docker Hub
-        if: github.event_name != 'pull_request'
-        uses: docker/login-action@v1
-        with:
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_PASSWORD }}
-      -
-        name: Login to GHCR
-        if: github.event_name != 'pull_request'
-        uses: docker/login-action@v1
-        with:
-          registry: ghcr.io
-          username: ${{ secrets.GHCR_USERNAME }}
-          password: ${{ secrets.GHCR_TOKEN }}
-      -
-        name: Build
-        uses: docker/build-push-action@v2
-        with:
-          context: ./docker
-          push: ${{ github.event_name != 'pull_request' }}
-          file: ./docker/Dockerfile.go_build
-          platforms: linux/amd64, linux/arm, linux/arm64, linux/386
-          tags: ${{ steps.docker_meta.outputs.tags }}
-          labels: ${{ steps.docker_meta.outputs.labels }}
-  build-large-release-container:
-    runs-on: [ubuntu-latest]
-
-    steps:
-      -
-        name: Checkout
-        uses: actions/checkout@v2
-      -
-        name: Docker meta
-        id: docker_meta
-        uses: docker/metadata-action@v3
-        with:
-          images: |
-            chrislusf/seaweedfs
-            ghcr.io/chrislusf/seaweedfs
-          tags: |
-            type=ref,event=tag,suffix=_large_disk
-          flavor: |
-            latest=false
-          labels: |
-            org.opencontainers.image.title=seaweedfs
-            org.opencontainers.image.description=SeaweedFS is a distributed storage system for blobs, objects, files, and data lake, to store and serve billions of files fast!
-            org.opencontainers.image.vendor=Chris Lu
-      -
-        name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
-      -
-        name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
-        with:
-          buildkitd-flags: "--debug"
-      -
-        name: Login to Docker Hub
-        if: github.event_name != 'pull_request'
-        uses: docker/login-action@v1
-        with:
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_PASSWORD }}
-      -
-        name: Login to GHCR
-        if: github.event_name != 'pull_request'
-        uses: docker/login-action@v1
-        with:
-          registry: ghcr.io
-          username: ${{ secrets.GHCR_USERNAME }}
-          password: ${{ secrets.GHCR_TOKEN }}
-      -
-        name: Build
-        uses: docker/build-push-action@v2
-        with:
-          context: ./docker
-          push: ${{ github.event_name != 'pull_request' }}
-          file: ./docker/Dockerfile.go_build_large
-          platforms: linux/amd64, linux/arm, linux/arm64, linux/386
-          tags: ${{ steps.docker_meta.outputs.tags }}
-          labels: ${{ steps.docker_meta.outputs.labels }}

.github/workflows/container_release1.yml

@@ -0,0 +1,54 @@
+name: "docker: build release containers for normal volume"
+
+on:
+  push:
+    tags:
+      - '*'
+  workflow_dispatch: []
+
+jobs:
+  build-default-release-container:
+    runs-on: [ubuntu-latest]
+
+    steps:
+      -
+        name: Checkout
+        uses: actions/checkout@v2
+      -
+        name: Docker meta
+        id: docker_meta
+        uses: docker/metadata-action@v3
+        with:
+          images: |
+            chrislusf/seaweedfs
+          tags: |
+            type=ref,event=tag
+          flavor: |
+            latest=false
+          labels: |
+            org.opencontainers.image.title=seaweedfs
+            org.opencontainers.image.description=SeaweedFS is a distributed storage system for blobs, objects, files, and data lake, to store and serve billions of files fast!
+            org.opencontainers.image.vendor=Chris Lu
+      -
+        name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+      -
+        name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+      -
+        name: Login to Docker Hub
+        if: github.event_name != 'pull_request'
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+      -
+        name: Build
+        uses: docker/build-push-action@v2
+        with:
+          context: ./docker
+          push: ${{ github.event_name != 'pull_request' }}
+          file: ./docker/Dockerfile.go_build
+          platforms: linux/amd64, linux/arm, linux/arm64, linux/386
+          tags: ${{ steps.docker_meta.outputs.tags }}
+          labels: ${{ steps.docker_meta.outputs.labels }}

.github/workflows/container_release2.yml

@@ -0,0 +1,55 @@
+name: "docker: build release containers for large volume"
+
+on:
+  push:
+    tags:
+      - '*'
+  workflow_dispatch: []
+
+jobs:
+
+  build-large-release-container:
+    runs-on: [ubuntu-latest]
+
+    steps:
+      -
+        name: Checkout
+        uses: actions/checkout@v2
+      -
+        name: Docker meta
+        id: docker_meta
+        uses: docker/metadata-action@v3
+        with:
+          images: |
+            chrislusf/seaweedfs
+          tags: |
+            type=ref,event=tag,suffix=_large_disk
+          flavor: |
+            latest=false
+          labels: |
+            org.opencontainers.image.title=seaweedfs
+            org.opencontainers.image.description=SeaweedFS is a distributed storage system for blobs, objects, files, and data lake, to store and serve billions of files fast!
+            org.opencontainers.image.vendor=Chris Lu
+      -
+        name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+      -
+        name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+      -
+        name: Login to Docker Hub
+        if: github.event_name != 'pull_request'
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+      -
+        name: Build
+        uses: docker/build-push-action@v2
+        with:
+          context: ./docker
+          push: ${{ github.event_name != 'pull_request' }}
+          file: ./docker/Dockerfile.go_build_large
+          platforms: linux/amd64, linux/arm, linux/arm64, linux/386
+          tags: ${{ steps.docker_meta.outputs.tags }}
+          labels: ${{ steps.docker_meta.outputs.labels }}

.github/workflows/container_release3.yml

@@ -0,0 +1,55 @@
+name: "docker: build release containers for rocksdb"
+
+on:
+  push:
+    tags:
+      - '*'
+  workflow_dispatch: []
+
+jobs:
+
+  build-large-release-container_rocksdb:
+    runs-on: [ubuntu-latest]
+
+    steps:
+      -
+        name: Checkout
+        uses: actions/checkout@v2
+      -
+        name: Docker meta
+        id: docker_meta
+        uses: docker/metadata-action@v3
+        with:
+          images: |
+            chrislusf/seaweedfs
+          tags: |
+            type=ref,event=tag,suffix=_large_disk_rocksdb
+          flavor: |
+            latest=false
+          labels: |
+            org.opencontainers.image.title=seaweedfs
+            org.opencontainers.image.description=SeaweedFS is a distributed storage system for blobs, objects, files, and data lake, to store and serve billions of files fast!
+            org.opencontainers.image.vendor=Chris Lu
+      -
+        name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+      -
+        name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+      -
+        name: Login to Docker Hub
+        if: github.event_name != 'pull_request'
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+      -
+        name: Build
+        uses: docker/build-push-action@v2
+        with:
+          context: ./docker
+          push: ${{ github.event_name != 'pull_request' }}
+          file: ./docker/Dockerfile.rocksdb_large
+          platforms: linux/amd64
+          tags: ${{ steps.docker_meta.outputs.tags }}
+          labels: ${{ steps.docker_meta.outputs.labels }}

README.md

@@ -147,6 +147,7 @@ Faster and Cheaper than direct cloud storage!
 * [AES256-GCM Encrypted Storage][FilerDataEncryption] safely stores the encrypted data.
 * [Super Large Files][SuperLargeFiles] stores large or super large files in tens of TB.
 * [Cloud Drive][CloudDrive] mounts cloud storage to local cluster, cached for fast read and write with asynchronous write back.
+* [Gateway to Remote Object Store][GatewayToRemoteObjectStore] mirrors bucket operations to remote object storage, in addition to [Cloud Drive][CloudDrive]
 
 ## Kubernetes ##
 * [Kubernetes CSI Driver][SeaweedFsCsiDriver] A Container Storage Interface (CSI) Driver. [![Docker Pulls](https://img.shields.io/docker/pulls/chrislusf/seaweedfs-csi-driver.svg?maxAge=4800)](https://hub.docker.com/r/chrislusf/seaweedfs-csi-driver/)
@@ -170,6 +171,7 @@ Faster and Cheaper than direct cloud storage!
 [FilerStoreReplication]: https://github.com/chrislusf/seaweedfs/wiki/Filer-Store-Replication
 [KeyLargeValueStore]: https://github.com/chrislusf/seaweedfs/wiki/Filer-as-a-Key-Large-Value-Store
 [CloudDrive]: https://github.com/chrislusf/seaweedfs/wiki/Cloud-Drive-Architecture
+[GatewayToRemoteObjectStore]: https://github.com/chrislusf/seaweedfs/wiki/Gateway-to-Remote-Object-Storage
 
 
 [Back to TOC](#table-of-contents)

docker/Dockerfile.rocksdb_large

@@ -0,0 +1,59 @@
+FROM golang:1.17-buster as builder
+
+RUN apt-get update
+RUN apt-get install -y build-essential libsnappy-dev zlib1g-dev libbz2-dev libgflags-dev liblz4-dev libzstd-dev
+
+ENV ROCKSDB_VERSION v6.22.1
+
+# build RocksDB
+RUN cd /tmp && \
+    git clone https://github.com/facebook/rocksdb.git /tmp/rocksdb --depth 1 --single-branch --branch $ROCKSDB_VERSION && \
+    cd rocksdb && \
+    make static_lib && \
+    make install-static
+
+ENV CGO_CFLAGS "-I/tmp/rocksdb/include"
+ENV CGO_LDFLAGS "-L/tmp/rocksdb -lrocksdb -lstdc++ -lm -lz -lbz2 -lsnappy -llz4 -lzstd"
+
+# build SeaweedFS
+RUN mkdir -p /go/src/github.com/chrislusf/
+RUN git clone https://github.com/chrislusf/seaweedfs /go/src/github.com/chrislusf/seaweedfs
+ARG BRANCH=${BRANCH:-master}
+RUN cd /go/src/github.com/chrislusf/seaweedfs && git checkout $BRANCH
+RUN cd /go/src/github.com/chrislusf/seaweedfs/weed \
+  && export LDFLAGS="-X github.com/chrislusf/seaweedfs/weed/util.COMMIT=$(git rev-parse --short HEAD)" \
+  && go install -tags "5BytesOffset rocksdb" -ldflags "-extldflags -static ${LDFLAGS}"
+
+
+FROM alpine AS final
+LABEL author="Chris Lu"
+COPY --from=builder /go/bin/weed /usr/bin/
+RUN mkdir -p /etc/seaweedfs
+COPY --from=builder /go/src/github.com/chrislusf/seaweedfs/docker/filer.toml /etc/seaweedfs/filer.toml
+COPY --from=builder /go/src/github.com/chrislusf/seaweedfs/docker/entrypoint.sh /entrypoint.sh
+RUN apk add fuse snappy gflags
+
+# volume server gprc port
+EXPOSE 18080
+# volume server http port
+EXPOSE 8080
+# filer server gprc port
+EXPOSE 18888
+# filer server http port
+EXPOSE 8888
+# master server shared gprc port
+EXPOSE 19333
+# master server shared http port
+EXPOSE 9333
+# s3 server http port
+EXPOSE 8333
+# webdav server http port
+EXPOSE 7333
+
+RUN mkdir -p /data/filerldb2
+
+VOLUME /data
+
+RUN chmod +x /entrypoint.sh
+
+ENTRYPOINT ["/entrypoint.sh"]

docker/Makefile

@@ -13,6 +13,12 @@ build: binary
 	docker build --no-cache -t chrislusf/seaweedfs:local -f Dockerfile.local .
 	rm ./weed
 
+build_gorocksdb:
+	docker build --no-cache -t chrislusf/gorocksdb -f Dockerfile.go_rocksdb .
+
+build_rocksdb:
+	docker build --no-cache -t chrislusf/seaweedfs:rocksdb -f Dockerfile.rocksdb_large .
+
 s3tests_build:
 	docker build --no-cache -t chrislusf/ceph-s3-tests:local -f Dockerfile.s3tests .
 

go.mod

@@ -8,7 +8,6 @@ require (
 	cloud.google.com/go/storage v1.16.1
 	github.com/Azure/azure-pipeline-go v0.2.3
 	github.com/Azure/azure-storage-blob-go v0.14.0
-	github.com/BurntSushi/toml v0.3.1 // indirect
 	github.com/DataDog/zstd v1.3.6-0.20190409195224-796139022798 // indirect
 	github.com/OneOfOne/xxhash v1.2.2
 	github.com/Shopify/sarama v1.23.1
@@ -39,6 +38,7 @@ require (
 	github.com/fsnotify/fsnotify v1.4.9 // indirect
 	github.com/go-errors/errors v1.1.1 // indirect
 	github.com/go-redis/redis/v8 v8.4.4
+	github.com/go-redsync/redsync/v4 v4.4.1
 	github.com/go-sql-driver/mysql v1.5.0
 	github.com/go-stack/stack v1.8.0 // indirect
 	github.com/go-zookeeper/zk v1.0.2 // indirect
@@ -54,11 +54,10 @@ require (
 	github.com/googleapis/gax-go v2.0.2+incompatible // indirect
 	github.com/googleapis/gax-go/v2 v2.1.0 // indirect
 	github.com/gorilla/mux v1.7.4
-	github.com/gorilla/websocket v1.4.1 // indirect
 	github.com/grpc-ecosystem/go-grpc-middleware v1.1.0
 	github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed // indirect
 	github.com/hashicorp/errwrap v1.0.0 // indirect
-	github.com/hashicorp/go-multierror v1.0.0 // indirect
+	github.com/hashicorp/go-multierror v1.1.0 // indirect
 	github.com/hashicorp/go-uuid v1.0.2 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
 	github.com/jcmturner/gofork v1.0.0 // indirect
@@ -74,6 +73,7 @@ require (
 	github.com/konsorten/go-windows-terminal-sequences v1.0.3 // indirect
 	github.com/kurin/blazer v0.5.3
 	github.com/lib/pq v1.10.0
+	github.com/linxGnu/grocksdb v1.6.38
 	github.com/magiconair/properties v1.8.1 // indirect
 	github.com/mailru/easyjson v0.7.1 // indirect
 	github.com/mattn/go-ieproxy v0.0.1 // indirect
@@ -110,12 +110,11 @@ require (
 	github.com/spf13/viper v1.4.0
 	github.com/streadway/amqp v0.0.0-20200108173154-1c71cc93ed71
 	github.com/stretchr/testify v1.7.0
+	github.com/stvp/tempredis v0.0.0-20181119212430-b82af8480203
 	github.com/syndtr/goleveldb v1.0.1-0.20190318030020-c3a204f8e965
-	github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c
 	github.com/tidwall/gjson v1.8.1
 	github.com/tidwall/match v1.0.3
 	github.com/tidwall/pretty v1.1.0 // indirect
-	github.com/tikv/client-go/v2 v2.0.0-alpha.0.20210824090536-16d902a3c7e5
 	github.com/tsuna/gohbase v0.0.0-20201125011725-348991136365
 	github.com/tylertreat/BoomFilters v0.0.0-20210315201527-1a82519a3e43
 	github.com/valyala/bytebufferpool v1.0.0
@@ -126,24 +125,24 @@ require (
 	github.com/xdg-go/scram v1.0.2 // indirect
 	github.com/xdg-go/stringprep v1.0.2 // indirect
 	github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect
-	go.etcd.io/etcd v3.3.25+incompatible
+	go.etcd.io/etcd/client/v3 v3.5.0
 	go.mongodb.org/mongo-driver v1.7.0
 	go.opencensus.io v0.23.0 // indirect
 	go.opentelemetry.io/otel v0.15.0 // indirect
 	gocloud.dev v0.20.0
 	gocloud.dev/pubsub/natspubsub v0.20.0
 	gocloud.dev/pubsub/rabbitpubsub v0.20.0
-	golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0 // indirect
+	golang.org/x/crypto v0.0.0-20201216223049-8b5274cf687f // indirect
 	golang.org/x/image v0.0.0-20200119044424-58c23975cae1
 	golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d
 	golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f // indirect
-	golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf
+	golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365
 	golang.org/x/text v0.3.6 // indirect
 	golang.org/x/tools v0.1.5
 	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
-	google.golang.org/api v0.56.0
+	google.golang.org/api v0.57.0
 	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/genproto v0.0.0-20210903162649-d08c68adba83 // indirect
+	google.golang.org/genproto v0.0.0-20210921142501-181ce0d877f6 // indirect
 	google.golang.org/grpc v1.40.0
 	google.golang.org/protobuf v1.27.1
 	gopkg.in/inf.v0 v0.9.1 // indirect
@@ -165,35 +164,23 @@ require (
 )
 
 require (
-	github.com/coreos/etcd v3.3.10+incompatible // indirect
-	github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548 // indirect
+	cloud.google.com/go/kms v1.0.0 // indirect
 	github.com/d4l3k/messagediff v1.2.1 // indirect
-	github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/jcmturner/aescts/v2 v2.0.0 // indirect
 	github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect
 	github.com/jcmturner/goidentity/v6 v6.0.1 // indirect
 	github.com/jcmturner/rpc/v2 v2.0.2 // indirect
 	github.com/mattn/go-runewidth v0.0.7 // indirect
-	github.com/opentracing/opentracing-go v1.2.0 // indirect
-	github.com/pingcap/errors v0.11.5-0.20201126102027-b0a155152ca3 // indirect
-	github.com/pingcap/failpoint v0.0.0-20210316064728-7acb0f0a3dfd // indirect
-	github.com/pingcap/kvproto v0.0.0-20210806074406-317f69fb54b4 // indirect
-	github.com/pingcap/log v0.0.0-20210317133921-96f4fcab92a4 // indirect
-	github.com/pingcap/parser v0.0.0-20210525032559-c37778aff307 // indirect
+	github.com/mattn/go-sqlite3 v2.0.1+incompatible // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
-	github.com/tikv/pd v1.1.0-beta.0.20210323121136-78679e5e209d // indirect
-	github.com/twmb/murmur3 v1.1.3 // indirect
 	go.etcd.io/etcd/api/v3 v3.5.0 // indirect
 	go.etcd.io/etcd/client/pkg/v3 v3.5.0 // indirect
-	go.etcd.io/etcd/client/v3 v3.5.0 // indirect
 	go.uber.org/atomic v1.7.0 // indirect
 	go.uber.org/multierr v1.7.0 // indirect
 	go.uber.org/zap v1.17.0 // indirect
-	golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 // indirect
 	golang.org/x/mod v0.4.2 // indirect
 	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
-	gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
 )

k8s/helm_charts2/Chart.yaml

@@ -1,5 +1,5 @@
 apiVersion: v1
 description: SeaweedFS
 name: seaweedfs
-appVersion: "2.67"
-version: "2.67"
+appVersion: "2.71"
+version: "2.71"

k8s/helm_charts2/templates/filer-servicemonitor.yaml

@@ -1,4 +1,5 @@
 {{- if .Values.filer.metricsPort }}
+{{- if .Values.global.monitoring.enabled }}
 apiVersion: monitoring.coreos.com/v1
 kind: ServiceMonitor
 metadata:
@@ -15,4 +16,5 @@ spec:
   selector:
     app: {{ template "seaweedfs.name" . }}
     component: filer
+{{- end }}
 {{- end }}

k8s/helm_charts2/templates/ingress.yaml

@@ -2,6 +2,7 @@ apiVersion: extensions/v1beta1
 kind: Ingress
 metadata:
  name: ingress-{{ template "seaweedfs.name" . }}-filer
+ namespace: {{ .Release.Namespace }}
  annotations:
    kubernetes.io/ingress.class: "nginx"
    nginx.ingress.kubernetes.io/auth-type: "basic"
@@ -32,6 +33,7 @@ apiVersion: extensions/v1beta1
 kind: Ingress
 metadata:
   name: ingress-{{ template "seaweedfs.name" . }}-master
+  namespace: {{ .Release.Namespace }}
   annotations:
     kubernetes.io/ingress.class: "nginx"
     nginx.ingress.kubernetes.io/auth-type: "basic"

k8s/helm_charts2/templates/s3-servicemonitor.yaml

@@ -1,4 +1,5 @@
 {{- if .Values.s3.metricsPort }}
+{{- if .Values.global.monitoring.enabled }}
 apiVersion: monitoring.coreos.com/v1
 kind: ServiceMonitor
 metadata:
@@ -15,4 +16,5 @@ spec:
   selector:
     app: {{ template "seaweedfs.name" . }}
     component: s3
+{{- end }}
 {{- end }}

k8s/helm_charts2/templates/volume-servicemonitor.yaml

@@ -1,4 +1,5 @@
 {{- if .Values.volume.metricsPort }}
+{{- if .Values.global.monitoring.enabled }}
 apiVersion: monitoring.coreos.com/v1
 kind: ServiceMonitor
 metadata:
@@ -15,4 +16,5 @@ spec:
   selector:
     app: {{ template "seaweedfs.name" . }}
     component: volume
+{{- end }}
 {{- end }}

k8s/helm_charts2/values.yaml

@@ -161,7 +161,7 @@ volume:
 
   # Directories to store data files. dir[,dir]... (default "/tmp")
   dir: "/data"
-  # Directories to store index files. dir[,dir]... (default "/tmp")
+  # Directories to store index files. dir[,dir]... (default is the same as "dir")
   dir_idx: null
 
   # Maximum numbers of volumes, count[,count]...

other/java/client/pom.xml

@@ -5,7 +5,7 @@
 
     <groupId>com.github.chrislusf</groupId>
     <artifactId>seaweedfs-client</artifactId>
-    <version>1.6.7</version>
+    <version>1.6.8</version>
 
     <parent>
         <groupId>org.sonatype.oss</groupId>
@@ -135,7 +135,7 @@
             <plugin>
                 <groupId>org.sonatype.plugins</groupId>
                 <artifactId>nexus-staging-maven-plugin</artifactId>
-                <version>1.6.7</version>
+                <version>1.6.8</version>
                 <extensions>true</extensions>
                 <configuration>
                     <serverId>ossrh</serverId>

other/java/client/pom.xml.deploy

@@ -5,7 +5,7 @@
 
     <groupId>com.github.chrislusf</groupId>
     <artifactId>seaweedfs-client</artifactId>
-    <version>1.6.7</version>
+    <version>1.6.8</version>
 
     <parent>
         <groupId>org.sonatype.oss</groupId>
@@ -130,7 +130,7 @@
             <plugin>
                 <groupId>org.sonatype.plugins</groupId>
                 <artifactId>nexus-staging-maven-plugin</artifactId>
-                <version>1.6.7</version>
+                <version>1.6.8</version>
                 <extensions>true</extensions>
                 <configuration>
                     <serverId>ossrh</serverId>

other/java/client/pom_debug.xml

@@ -5,7 +5,7 @@
 
     <groupId>com.github.chrislusf</groupId>
     <artifactId>seaweedfs-client</artifactId>
-    <version>1.6.7</version>
+    <version>1.6.8</version>
 
     <parent>
         <groupId>org.sonatype.oss</groupId>

other/java/client/src/main/java/seaweedfs/client/ChunkCache.java

@@ -15,7 +15,7 @@ public class ChunkCache {
         }
         this.cache = CacheBuilder.newBuilder()
                 .maximumSize(maxEntries)
-                .expireAfterAccess(1, TimeUnit.HOURS)
+                .expireAfterWrite(1, TimeUnit.HOURS)
                 .build();
     }
 

other/java/client/src/main/java/seaweedfs/client/FilerClient.java

@@ -14,7 +14,11 @@ public class FilerClient extends FilerGrpcClient {
     private static final Logger LOG = LoggerFactory.getLogger(FilerClient.class);
 
     public FilerClient(String host, int grpcPort) {
-        super(host, grpcPort);
+        super(host, grpcPort-10000, grpcPort);
+    }
+
+    public FilerClient(String host, int port, int grpcPort) {
+        super(host, port, grpcPort);
     }
 
     public static String toFileId(FilerProto.FileId fid) {

other/java/client/src/main/java/seaweedfs/client/FilerGrpcClient.java

@@ -40,11 +40,11 @@ public class FilerGrpcClient {
     private int volumeServerAccess = VOLUME_SERVER_ACCESS_DIRECT;
     private String filerAddress;
 
-    public FilerGrpcClient(String host, int grpcPort) {
-        this(host, grpcPort, sslContext);
+    public FilerGrpcClient(String host, int port, int grpcPort) {
+        this(host, port, grpcPort, sslContext);
     }
 
-    public FilerGrpcClient(String host, int grpcPort, SslContext sslContext) {
+    public FilerGrpcClient(String host, int port, int grpcPort, SslContext sslContext) {
 
         this(sslContext == null ?
                 ManagedChannelBuilder.forAddress(host, grpcPort).usePlaintext()
@@ -54,7 +54,7 @@ public class FilerGrpcClient {
                         .negotiationType(NegotiationType.TLS)
                         .sslContext(sslContext));
 
-        filerAddress = SeaweedUtil.joinHostPort(host, grpcPort - 10000);
+        filerAddress = SeaweedUtil.joinHostPort(host, port);
 
         FilerProto.GetFilerConfigurationResponse filerConfigurationResponse =
                 this.getBlockingStub().getFilerConfiguration(

other/java/client/src/main/java/seaweedfs/client/SeaweedWrite.java

@@ -59,7 +59,7 @@ public class SeaweedWrite {
         String fileId = response.getFileId();
         String auth = response.getAuth();
 
-        String targetUrl = filerClient.getChunkUrl(fileId, response.getUrl(), response.getPublicUrl());
+        String targetUrl = filerClient.getChunkUrl(fileId, response.getLocation().getUrl(), response.getLocation().getPublicUrl());
 
         ByteString cipherKeyString = com.google.protobuf.ByteString.EMPTY;
         byte[] cipherKey = null;

other/java/client/src/main/java/seaweedfs/client/VolumeIdCache.java

@@ -15,7 +15,7 @@ public class VolumeIdCache {
         }
         this.cache = CacheBuilder.newBuilder()
                 .maximumSize(maxEntries)
-                .expireAfterAccess(5, TimeUnit.MINUTES)
+                .expireAfterWrite(5, TimeUnit.MINUTES)
                 .build();
     }
 

other/java/client/src/main/proto/filer.proto

@@ -238,13 +238,12 @@ message AssignVolumeRequest {
 
 message AssignVolumeResponse {
     string file_id = 1;
-    string url = 2;
-    string public_url = 3;
     int32 count = 4;
     string auth = 5;
     string collection = 6;
     string replication = 7;
     string error = 8;
+    Location location = 9;
 }
 
 message LookupVolumeRequest {
@@ -258,6 +257,7 @@ message Locations {
 message Location {
     string url = 1;
     string public_url = 2;
+    uint32 grpc_port = 3;
 }
 message LookupVolumeResponse {
     map<string, Locations> locations_map = 1;

other/java/examples/pom.xml

@@ -11,13 +11,13 @@
         <dependency>
             <groupId>com.github.chrislusf</groupId>
             <artifactId>seaweedfs-client</artifactId>
-            <version>1.6.7</version>
+            <version>1.6.8</version>
             <scope>compile</scope>
         </dependency>
         <dependency>
             <groupId>com.github.chrislusf</groupId>
             <artifactId>seaweedfs-hadoop2-client</artifactId>
-            <version>1.6.7</version>
+            <version>1.6.8</version>
             <scope>compile</scope>
         </dependency>
         <dependency>

other/java/hdfs2/dependency-reduced-pom.xml

@@ -86,7 +86,7 @@
       <plugin>
         <groupId>org.sonatype.plugins</groupId>
         <artifactId>nexus-staging-maven-plugin</artifactId>
-        <version>1.6.7</version>
+        <version>1.6.8</version>
         <extensions>true</extensions>
         <configuration>
           <serverId>ossrh</serverId>
@@ -301,7 +301,7 @@
     </snapshotRepository>
   </distributionManagement>
   <properties>
-    <seaweedfs.client.version>1.6.7</seaweedfs.client.version>
+    <seaweedfs.client.version>1.6.8</seaweedfs.client.version>
     <hadoop.version>2.9.2</hadoop.version>
   </properties>
 </project>

other/java/hdfs2/pom.xml

@@ -5,7 +5,7 @@
     <modelVersion>4.0.0</modelVersion>
 
     <properties>
-        <seaweedfs.client.version>1.6.7</seaweedfs.client.version>
+        <seaweedfs.client.version>1.6.8</seaweedfs.client.version>
         <hadoop.version>2.9.2</hadoop.version>
     </properties>
 
@@ -105,7 +105,7 @@
             <plugin>
                 <groupId>org.sonatype.plugins</groupId>
                 <artifactId>nexus-staging-maven-plugin</artifactId>
-                <version>1.6.7</version>
+                <version>1.6.8</version>
                 <extensions>true</extensions>
                 <configuration>
                     <serverId>ossrh</serverId>

other/java/hdfs2/src/main/java/seaweed/hdfs/SeaweedFileSystem.java

@@ -23,6 +23,7 @@ public class SeaweedFileSystem extends FileSystem {
 
     public static final String FS_SEAWEED_FILER_HOST = "fs.seaweed.filer.host";
     public static final String FS_SEAWEED_FILER_PORT = "fs.seaweed.filer.port";
+    public static final String FS_SEAWEED_FILER_PORT_GRPC = "fs.seaweed.filer.port.grpc";
     public static final int FS_SEAWEED_DEFAULT_PORT = 8888;
     public static final String FS_SEAWEED_BUFFER_SIZE = "fs.seaweed.buffer.size";
     public static final String FS_SEAWEED_REPLICATION = "fs.seaweed.replication";
@@ -50,9 +51,6 @@ public class SeaweedFileSystem extends FileSystem {
         // get host information from uri (overrides info in conf)
         String host = uri.getHost();
         host = (host == null) ? conf.get(FS_SEAWEED_FILER_HOST, "localhost") : host;
-        if (host == null) {
-            throw new IOException("Invalid host specified");
-        }
         conf.set(FS_SEAWEED_FILER_HOST, host);
 
         // get port information from uri, (overrides info in conf)
@@ -60,10 +58,12 @@ public class SeaweedFileSystem extends FileSystem {
         port = (port == -1) ? FS_SEAWEED_DEFAULT_PORT : port;
         conf.setInt(FS_SEAWEED_FILER_PORT, port);
 
+        int grpcPort = conf.getInt(FS_SEAWEED_FILER_PORT_GRPC, port+10000);
+
         setConf(conf);
         this.uri = uri;
 
-        seaweedFileSystemStore = new SeaweedFileSystemStore(host, port, conf);
+        seaweedFileSystemStore = new SeaweedFileSystemStore(host, port, grpcPort, conf);
 
     }
 

other/java/hdfs2/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java

@@ -27,9 +27,8 @@ public class SeaweedFileSystemStore {
     private FilerClient filerClient;
     private Configuration conf;
 
-    public SeaweedFileSystemStore(String host, int port, Configuration conf) {
-        int grpcPort = 10000 + port;
-        filerClient = new FilerClient(host, grpcPort);
+    public SeaweedFileSystemStore(String host, int port, int grpcPort, Configuration conf) {
+        filerClient = new FilerClient(host, port, grpcPort);
         this.conf = conf;
         String volumeServerAccessMode = this.conf.get(FS_SEAWEED_VOLUME_SERVER_ACCESS, "direct");
         if (volumeServerAccessMode.equals("publicUrl")) {

other/java/hdfs3/dependency-reduced-pom.xml

@@ -86,7 +86,7 @@
       <plugin>
         <groupId>org.sonatype.plugins</groupId>
         <artifactId>nexus-staging-maven-plugin</artifactId>
-        <version>1.6.7</version>
+        <version>1.6.8</version>
         <extensions>true</extensions>
         <configuration>
           <serverId>ossrh</serverId>
@@ -309,7 +309,7 @@
     </snapshotRepository>
   </distributionManagement>
   <properties>
-    <seaweedfs.client.version>1.6.7</seaweedfs.client.version>
+    <seaweedfs.client.version>1.6.8</seaweedfs.client.version>
     <hadoop.version>3.1.1</hadoop.version>
   </properties>
 </project>

other/java/hdfs3/pom.xml

@@ -5,7 +5,7 @@
     <modelVersion>4.0.0</modelVersion>
 
     <properties>
-        <seaweedfs.client.version>1.6.7</seaweedfs.client.version>
+        <seaweedfs.client.version>1.6.8</seaweedfs.client.version>
         <hadoop.version>3.1.1</hadoop.version>
     </properties>
 
@@ -105,7 +105,7 @@
             <plugin>
                 <groupId>org.sonatype.plugins</groupId>
                 <artifactId>nexus-staging-maven-plugin</artifactId>
-                <version>1.6.7</version>
+                <version>1.6.8</version>
                 <extensions>true</extensions>
                 <configuration>
                     <serverId>ossrh</serverId>

other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystem.java

@@ -23,6 +23,7 @@ public class SeaweedFileSystem extends FileSystem {
 
     public static final String FS_SEAWEED_FILER_HOST = "fs.seaweed.filer.host";
     public static final String FS_SEAWEED_FILER_PORT = "fs.seaweed.filer.port";
+    public static final String FS_SEAWEED_FILER_PORT_GRPC = "fs.seaweed.filer.port.grpc";
     public static final int FS_SEAWEED_DEFAULT_PORT = 8888;
     public static final String FS_SEAWEED_BUFFER_SIZE = "fs.seaweed.buffer.size";
     public static final String FS_SEAWEED_REPLICATION = "fs.seaweed.replication";
@@ -50,9 +51,6 @@ public class SeaweedFileSystem extends FileSystem {
         // get host information from uri (overrides info in conf)
         String host = uri.getHost();
         host = (host == null) ? conf.get(FS_SEAWEED_FILER_HOST, "localhost") : host;
-        if (host == null) {
-            throw new IOException("Invalid host specified");
-        }
         conf.set(FS_SEAWEED_FILER_HOST, host);
 
         // get port information from uri, (overrides info in conf)
@@ -60,10 +58,12 @@ public class SeaweedFileSystem extends FileSystem {
         port = (port == -1) ? FS_SEAWEED_DEFAULT_PORT : port;
         conf.setInt(FS_SEAWEED_FILER_PORT, port);
 
+        int grpcPort = conf.getInt(FS_SEAWEED_FILER_PORT_GRPC, port+10000);
+
         setConf(conf);
         this.uri = uri;
 
-        seaweedFileSystemStore = new SeaweedFileSystemStore(host, port, conf);
+        seaweedFileSystemStore = new SeaweedFileSystemStore(host, port, grpcPort, conf);
 
     }
 

other/java/hdfs3/src/main/java/seaweed/hdfs/SeaweedFileSystemStore.java

@@ -27,9 +27,8 @@ public class SeaweedFileSystemStore {
     private FilerClient filerClient;
     private Configuration conf;
 
-    public SeaweedFileSystemStore(String host, int port, Configuration conf) {
-        int grpcPort = 10000 + port;
-        filerClient = new FilerClient(host, grpcPort);
+    public SeaweedFileSystemStore(String host, int port, int grpcPort, Configuration conf) {
+        filerClient = new FilerClient(host, port, grpcPort);
         this.conf = conf;
         String volumeServerAccessMode = this.conf.get(FS_SEAWEED_VOLUME_SERVER_ACCESS, "direct");
         if (volumeServerAccessMode.equals("publicUrl")) {

test/s3/multipart/aws_upload.go

@@ -19,10 +19,10 @@ import (
 const (
 	maxPartSize        = int64(5 * 1024 * 1024)
 	maxRetries         = 3
-	awsAccessKeyID     = "Your access key"
-	awsSecretAccessKey = "Your secret key"
-	awsBucketRegion    = "S3 bucket region"
-	awsBucketName      = "newBucket"
+	awsAccessKeyID     = "any"
+	awsSecretAccessKey = "any"
+	awsBucketRegion    = "us‑west‑1"
+	awsBucketName      = "bucket1"
 )
 
 var (
@@ -37,7 +37,7 @@ func main() {
 	if err != nil {
 		fmt.Printf("bad credentials: %s", err)
 	}
-	cfg := aws.NewConfig().WithRegion(awsBucketRegion).WithCredentials(creds).WithDisableSSL(true).WithEndpoint("localhost:8333")
+	cfg := aws.NewConfig().WithRegion(awsBucketRegion).WithCredentials(creds).WithDisableSSL(true).WithEndpoint("localhost:8333").WithS3ForcePathStyle(true)
 	svc := s3.New(session.New(), cfg)
 
 	file, err := os.Open(*filename)

unmaintained/diff_volume_servers/diff_volume_servers.go

@@ -6,13 +6,9 @@ import (
 	"errors"
 	"flag"
 	"fmt"
-	"io"
-	"math"
-	"os"
-	"strings"
-
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/operation"
+	"github.com/chrislusf/seaweedfs/weed/pb"
 	"github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb"
 	"github.com/chrislusf/seaweedfs/weed/security"
 	"github.com/chrislusf/seaweedfs/weed/storage/idx"
@@ -20,6 +16,9 @@ import (
 	"github.com/chrislusf/seaweedfs/weed/storage/types"
 	"github.com/chrislusf/seaweedfs/weed/util"
 	"google.golang.org/grpc"
+	"io"
+	"math"
+	"os"
 )
 
 var (
@@ -45,13 +44,13 @@ func main() {
 	grpcDialOption = security.LoadClientTLS(util.GetViper(), "grpc.client")
 
 	vid := uint32(*volumeId)
-	servers := strings.Split(*serversStr, ",")
+	servers := pb.ServerAddresses(*serversStr).ToAddresses()
 	if len(servers) < 2 {
 		glog.Fatalf("You must specify more than 1 server\n")
 	}
-	var referenceServer string
+	var referenceServer pb.ServerAddress
 	var maxOffset int64
-	allFiles := map[string]map[types.NeedleId]needleState{}
+	allFiles := map[pb.ServerAddress]map[types.NeedleId]needleState{}
 	for _, addr := range servers {
 		files, offset, err := getVolumeFiles(vid, addr)
 		if err != nil {
@@ -121,7 +120,7 @@ type needleState struct {
 	size  types.Size
 }
 
-func getVolumeFiles(v uint32, addr string) (map[types.NeedleId]needleState, int64, error) {
+func getVolumeFiles(v uint32, addr pb.ServerAddress) (map[types.NeedleId]needleState, int64, error) {
 	var idxFile *bytes.Reader
 	err := operation.WithVolumeServerClient(addr, grpcDialOption, func(vs volume_server_pb.VolumeServerClient) error {
 		ctx, cancel := context.WithCancel(context.Background())
@@ -179,7 +178,7 @@ func getVolumeFiles(v uint32, addr string) (map[types.NeedleId]needleState, int6
 	return files, maxOffset, nil
 }
 
-func getNeedleFileId(v uint32, nid types.NeedleId, addr string) (string, error) {
+func getNeedleFileId(v uint32, nid types.NeedleId, addr pb.ServerAddress) (string, error) {
 	var id string
 	err := operation.WithVolumeServerClient(addr, grpcDialOption, func(vs volume_server_pb.VolumeServerClient) error {
 		resp, err := vs.VolumeNeedleStatus(context.Background(), &volume_server_pb.VolumeNeedleStatusRequest{

unmaintained/load_test/load_test_meta_tail/load_test_meta_tail.go

@@ -51,7 +51,7 @@ func main() {
 }
 
 func startGenerateMetadata() {
-	pb.WithFilerClient(*tailFiler, grpc.WithInsecure(), func(client filer_pb.SeaweedFilerClient) error {
+	pb.WithFilerClient(pb.ServerAddress(*tailFiler), grpc.WithInsecure(), func(client filer_pb.SeaweedFilerClient) error {
 
 		for i := 0; i < *n; i++ {
 			name := fmt.Sprintf("file%d", i)
@@ -77,7 +77,7 @@ func startGenerateMetadata() {
 
 func startSubscribeMetadata(eachEntryFunc func(event *filer_pb.SubscribeMetadataResponse) error) {
 
-	tailErr := pb.FollowMetadata(*tailFiler, grpc.WithInsecure(), "tail", *dir, nil, 0, 0, eachEntryFunc, false)
+	tailErr := pb.FollowMetadata(pb.ServerAddress(*tailFiler), grpc.WithInsecure(), "tail", *dir, nil, 0, 0, eachEntryFunc, false)
 
 	if tailErr != nil {
 		fmt.Printf("tail %s: %v\n", *tailFiler, tailErr)

unmaintained/repeated_vacuum/repeated_vacuum.go

@@ -3,6 +3,7 @@ package main
 import (
 	"flag"
 	"fmt"
+	"github.com/chrislusf/seaweedfs/weed/pb"
 	"log"
 	"math/rand"
 	"time"
@@ -32,7 +33,7 @@ func main() {
 	go func() {
 		for {
 			println("vacuum threshold", *garbageThreshold)
-			_, _, err := util.Get(fmt.Sprintf("http://%s/vol/vacuum?garbageThreshold=%f", *master, *garbageThreshold))
+			_, _, err := util.Get(fmt.Sprintf("http://%s/vol/vacuum?garbageThreshold=%f", pb.ServerAddress(*master).ToHttpAddress(), *garbageThreshold))
 			if err != nil {
 				log.Fatalf("vacuum: %v", err)
 			}
@@ -52,7 +53,7 @@ func main() {
 }
 
 func genFile(grpcDialOption grpc.DialOption, i int) (*operation.AssignResult, string) {
-	assignResult, err := operation.Assign(func() string { return *master }, grpcDialOption, &operation.VolumeAssignRequest{
+	assignResult, err := operation.Assign(func() pb.ServerAddress { return pb.ServerAddress(*master) }, grpcDialOption, &operation.VolumeAssignRequest{
 		Count:       1,
 		Replication: *replication,
 	})

unmaintained/stream_read_volume/stream_read_volume.go

@@ -0,0 +1,64 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"flag"
+	"fmt"
+	"github.com/chrislusf/seaweedfs/weed/operation"
+	"github.com/chrislusf/seaweedfs/weed/pb"
+	"github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb"
+	"github.com/chrislusf/seaweedfs/weed/security"
+	"github.com/chrislusf/seaweedfs/weed/util"
+	"google.golang.org/grpc"
+	"io"
+)
+
+var (
+	volumeServer       = flag.String("volumeServer", "localhost:8080", "a volume server")
+	volumeId         = flag.Int("volumeId", -1, "a volume id to stream read")
+	grpcDialOption   grpc.DialOption
+)
+
+func main() {
+	flag.Parse()
+
+	util.LoadConfiguration("security", false)
+	grpcDialOption = security.LoadClientTLS(util.GetViper(), "grpc.client")
+
+	vid := uint32(*volumeId)
+
+	eachNeedleFunc := func(resp *volume_server_pb.ReadAllNeedlesResponse) error {
+		fmt.Printf("%d,%x%08x %d\n", resp.VolumeId, resp.NeedleId, resp.Cookie, len(resp.NeedleBlob))
+		return nil
+	}
+
+	err := operation.WithVolumeServerClient(pb.ServerAddress(*volumeServer), grpcDialOption, func(vs volume_server_pb.VolumeServerClient) error {
+		ctx, cancel := context.WithCancel(context.Background())
+		defer cancel()
+		copyFileClient, err := vs.ReadAllNeedles(ctx, &volume_server_pb.ReadAllNeedlesRequest{
+			VolumeIds:           []uint32{vid},
+		})
+		if err != nil {
+			return err
+		}
+		for {
+			resp, err := copyFileClient.Recv()
+			if errors.Is(err, io.EOF) {
+				break
+			}
+			if err != nil {
+				return err
+			}
+			if err = eachNeedleFunc(resp); err != nil {
+				return err
+			}
+		}
+		return nil
+	})
+	if err != nil {
+		fmt.Printf("read %s: %v\n", *volumeServer, err)
+	}
+
+}
+

unmaintained/volume_tailer/volume_tailer.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"flag"
+	"github.com/chrislusf/seaweedfs/weed/pb"
 	"log"
 	"time"
 
@@ -37,7 +38,7 @@ func main() {
 		sinceTimeNs = time.Now().Add(-*rewindDuration).UnixNano()
 	}
 
-	err := operation.TailVolume(func()string{return *master}, grpcDialOption, vid, uint64(sinceTimeNs), *timeoutSeconds, func(n *needle.Needle) (err error) {
+	err := operation.TailVolume(func()pb.ServerAddress{return pb.ServerAddress(*master)}, grpcDialOption, vid, uint64(sinceTimeNs), *timeoutSeconds, func(n *needle.Needle) (err error) {
 		if n.Size == 0 {
 			println("-", n.String())
 			return nil

weed/command/backup.go

@@ -2,6 +2,7 @@ package command
 
 import (
 	"fmt"
+	"github.com/chrislusf/seaweedfs/weed/pb"
 
 	"github.com/chrislusf/seaweedfs/weed/security"
 	"github.com/chrislusf/seaweedfs/weed/storage/needle"
@@ -72,12 +73,12 @@ func runBackup(cmd *Command, args []string) bool {
 	vid := needle.VolumeId(*s.volumeId)
 
 	// find volume location, replication, ttl info
-	lookup, err := operation.LookupVolumeId(func() string { return *s.master }, grpcDialOption, vid.String())
+	lookup, err := operation.LookupVolumeId(func() pb.ServerAddress { return pb.ServerAddress(*s.master) }, grpcDialOption, vid.String())
 	if err != nil {
 		fmt.Printf("Error looking up volume %d: %v\n", vid, err)
 		return true
 	}
-	volumeServer := lookup.Locations[0].Url
+	volumeServer := lookup.Locations[0].ServerAddress()
 
 	stats, err := operation.GetVolumeSyncStatus(volumeServer, grpcDialOption, uint32(vid))
 	if err != nil {

weed/command/benchmark.go

@@ -3,6 +3,7 @@ package command
 import (
 	"bufio"
 	"fmt"
+	"github.com/chrislusf/seaweedfs/weed/pb"
 	"io"
 	"math"
 	"math/rand"
@@ -10,7 +11,6 @@ import (
 	"runtime"
 	"runtime/pprof"
 	"sort"
-	"strings"
 	"sync"
 	"time"
 
@@ -129,7 +129,7 @@ func runBenchmark(cmd *Command, args []string) bool {
 		defer pprof.StopCPUProfile()
 	}
 
-	b.masterClient = wdclient.NewMasterClient(b.grpcDialOption, "client", "", 0, "", strings.Split(*b.masters, ","))
+	b.masterClient = wdclient.NewMasterClient(b.grpcDialOption, "client", "", "", pb.ServerAddresses(*b.masters).ToAddresses())
 	go b.masterClient.KeepConnectedToMaster()
 	b.masterClient.WaitUntilConnected()
 

weed/command/command.go

@@ -21,6 +21,7 @@ var Commands = []*Command{
 	cmdFilerCopy,
 	cmdFilerMetaBackup,
 	cmdFilerMetaTail,
+	cmdFilerRemoteGateway,
 	cmdFilerRemoteSynchronize,
 	cmdFilerReplicate,
 	cmdFilerSynchronize,

weed/command/download.go

@@ -2,6 +2,7 @@ package command
 
 import (
 	"fmt"
+	"github.com/chrislusf/seaweedfs/weed/pb"
 	"github.com/chrislusf/seaweedfs/weed/security"
 	"google.golang.org/grpc"
 	"io"
@@ -49,7 +50,7 @@ func runDownload(cmd *Command, args []string) bool {
 	grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client")
 
 	for _, fid := range args {
-		if e := downloadToFile(func() string { return *d.server }, grpcDialOption, fid, util.ResolvePath(*d.dir)); e != nil {
+		if e := downloadToFile(func() pb.ServerAddress { return pb.ServerAddress(*d.server) }, grpcDialOption, fid, util.ResolvePath(*d.dir)); e != nil {
 			fmt.Println("Download Error: ", fid, e)
 		}
 	}

weed/command/filer.go

@@ -29,10 +29,12 @@ var (
 )
 
 type FilerOptions struct {
-	masters                 *string
+	masters                 []pb.ServerAddress
+	mastersString           *string
 	ip                      *string
 	bindIp                  *string
 	port                    *int
+	portGrpc                *int
 	publicPort              *int
 	collection              *string
 	defaultReplicaPlacement *string
@@ -55,11 +57,12 @@ type FilerOptions struct {
 
 func init() {
 	cmdFiler.Run = runFiler // break init cycle
-	f.masters = cmdFiler.Flag.String("master", "localhost:9333", "comma-separated master servers")
+	f.mastersString = cmdFiler.Flag.String("master", "localhost:9333", "comma-separated master servers")
 	f.collection = cmdFiler.Flag.String("collection", "", "all data will be stored in this default collection")
 	f.ip = cmdFiler.Flag.String("ip", util.DetectedHostAddress(), "filer server http listen ip address")
 	f.bindIp = cmdFiler.Flag.String("ip.bind", "", "ip address to bind to")
 	f.port = cmdFiler.Flag.Int("port", 8888, "filer server http listen port")
+	f.portGrpc = cmdFiler.Flag.Int("port.grpc", 0, "filer server grpc listen port")
 	f.publicPort = cmdFiler.Flag.Int("port.readonly", 0, "readonly port opened to public")
 	f.defaultReplicaPlacement = cmdFiler.Flag.String("defaultReplicaPlacement", "", "default replication type. If not specified, use master setting.")
 	f.disableDirListing = cmdFiler.Flag.Bool("disableDirListing", false, "turn off directory listing")
@@ -84,7 +87,7 @@ func init() {
 	filerS3Options.tlsPrivateKey = cmdFiler.Flag.String("s3.key.file", "", "path to the TLS private key file")
 	filerS3Options.tlsCertificate = cmdFiler.Flag.String("s3.cert.file", "", "path to the TLS certificate file")
 	filerS3Options.config = cmdFiler.Flag.String("s3.config", "", "path to the config file")
-	filerS3Options.allowEmptyFolder = cmdFiler.Flag.Bool("s3.allowEmptyFolder", false, "allow empty folders")
+	filerS3Options.allowEmptyFolder = cmdFiler.Flag.Bool("s3.allowEmptyFolder", true, "allow empty folders")
 
 	// start webdav on filer
 	filerStartWebDav = cmdFiler.Flag.Bool("webdav", false, "whether to start webdav gateway")
@@ -155,13 +158,15 @@ func runFiler(cmd *Command, args []string) bool {
 
 	if *filerStartIam {
 		filerIamOptions.filer = &filerAddress
-		filerIamOptions.masters = f.masters
+		filerIamOptions.masters = f.mastersString
 		go func() {
 			time.Sleep(startDelay * time.Second)
 			filerIamOptions.startIamServer()
 		}()
 	}
 
+	f.masters = pb.ServerAddresses(*f.mastersString).ToAddresses()
+
 	f.startFiler()
 
 	return true
@@ -175,6 +180,9 @@ func (fo *FilerOptions) startFiler() {
 	if *fo.publicPort != 0 {
 		publicVolumeMux = http.NewServeMux()
 	}
+	if *fo.portGrpc == 0 {
+		*fo.portGrpc = 10000 + *fo.port
+	}
 
 	defaultLevelDbDirectory := util.ResolvePath(*fo.defaultLevelDbDirectory + "/filerldb2")
 
@@ -183,8 +191,10 @@ func (fo *FilerOptions) startFiler() {
 		peers = strings.Split(*fo.peers, ",")
 	}
 
+	filerAddress := pb.NewServerAddress(*fo.ip, *fo.port, *fo.portGrpc)
+
 	fs, nfs_err := weed_server.NewFilerServer(defaultMux, publicVolumeMux, &weed_server.FilerOption{
-		Masters:               strings.Split(*fo.masters, ","),
+		Masters:               fo.masters,
 		Collection:            *fo.collection,
 		DefaultReplication:    *fo.defaultReplicaPlacement,
 		DisableDirListing:     *fo.disableDirListing,
@@ -194,11 +204,10 @@ func (fo *FilerOptions) startFiler() {
 		Rack:                  *fo.rack,
 		DefaultLevelDbDir:     defaultLevelDbDirectory,
 		DisableHttp:           *fo.disableHttp,
-		Host:                  *fo.ip,
-		Port:                  uint32(*fo.port),
+		Host:                  filerAddress,
 		Cipher:                *fo.cipher,
 		SaveToFilerLimit:      int64(*fo.saveToFilerLimit),
-		Filers:                peers,
+		Filers:                pb.FromAddressStrings(peers),
 		ConcurrentUploadLimit: int64(*fo.concurrentUploadLimitMB) * 1024 * 1024,
 	})
 	if nfs_err != nil {
@@ -229,7 +238,7 @@ func (fo *FilerOptions) startFiler() {
 	}
 
 	// starting grpc server
-	grpcPort := *fo.port + 10000
+	grpcPort := *fo.portGrpc
 	grpcL, err := util.NewListener(util.JoinHostPort(*fo.bindIp, grpcPort), 0)
 	if err != nil {
 		glog.Fatalf("failed to listen on grpc port %d: %v", grpcPort, err)

weed/command/filer_backup.go

@@ -78,7 +78,7 @@ func doFilerBackup(grpcDialOption grpc.DialOption, backupOption *FilerBackupOpti
 		return fmt.Errorf("no data sink configured in replication.toml")
 	}
 
-	sourceFiler := *backupOption.filer
+	sourceFiler := pb.ServerAddress(*backupOption.filer)
 	sourcePath := *backupOption.path
 	timeAgo := *backupOption.timeAgo
 	targetPath := dataSink.GetSinkToDirectory()
@@ -102,7 +102,7 @@ func doFilerBackup(grpcDialOption grpc.DialOption, backupOption *FilerBackupOpti
 
 	// create filer sink
 	filerSource := &source.FilerSource{}
-	filerSource.DoInitialize(sourceFiler, pb.ServerToGrpcAddress(sourceFiler), sourcePath, *backupOption.proxyByFiler)
+	filerSource.DoInitialize(sourceFiler.ToHttpAddress(), sourceFiler.ToGrpcAddress(), sourcePath, *backupOption.proxyByFiler)
 	dataSink.SetSourceFiler(filerSource)
 
 	processEventFn := genProcessFunction(sourcePath, targetPath, dataSink, debug)

weed/command/filer_cat.go

@@ -23,7 +23,7 @@ var (
 
 type FilerCatOptions struct {
 	grpcDialOption grpc.DialOption
-	filerAddress   string
+	filerAddress   pb.ServerAddress
 	filerClient    filer_pb.SeaweedFilerClient
 	output         *string
 }
@@ -78,7 +78,7 @@ func runFilerCat(cmd *Command, args []string) bool {
 		return false
 	}
 
-	filerCat.filerAddress = filerUrl.Host
+	filerCat.filerAddress = pb.ServerAddress(filerUrl.Host)
 	filerCat.grpcDialOption = security.LoadClientTLS(util.GetViper(), "grpc.client")
 
 	dir, name := util.FullPath(urlPath).DirAndName()

weed/command/filer_copy.go

@@ -7,7 +7,6 @@ import (
 	"io"
 	"io/ioutil"
 	"net/http"
-	"net/url"
 	"os"
 	"path/filepath"
 	"strconv"
@@ -92,35 +91,21 @@ func runCopy(cmd *Command, args []string) bool {
 	filerDestination := args[len(args)-1]
 	fileOrDirs := args[0 : len(args)-1]
 
-	filerUrl, err := url.Parse(filerDestination)
+	filerAddress, urlPath, err := pb.ParseUrl(filerDestination)
 	if err != nil {
 		fmt.Printf("The last argument should be a URL on filer: %v\n", err)
 		return false
 	}
-	urlPath := filerUrl.Path
 	if !strings.HasSuffix(urlPath, "/") {
 		fmt.Printf("The last argument should be a folder and end with \"/\"\n")
 		return false
 	}
 
-	if filerUrl.Port() == "" {
-		fmt.Printf("The filer port should be specified.\n")
-		return false
-	}
-
-	filerPort, parseErr := strconv.ParseUint(filerUrl.Port(), 10, 64)
-	if parseErr != nil {
-		fmt.Printf("The filer port parse error: %v\n", parseErr)
-		return false
-	}
-
-	filerGrpcPort := filerPort + 10000
-	filerGrpcAddress := util.JoinHostPort(filerUrl.Hostname(), int(filerGrpcPort))
 	copy.grpcDialOption = security.LoadClientTLS(util.GetViper(), "grpc.client")
 
-	masters, collection, replication, dirBuckets, maxMB, cipher, err := readFilerConfiguration(copy.grpcDialOption, filerGrpcAddress)
+	masters, collection, replication, dirBuckets, maxMB, cipher, err := readFilerConfiguration(copy.grpcDialOption, filerAddress)
 	if err != nil {
-		fmt.Printf("read from filer %s: %v\n", filerGrpcAddress, err)
+		fmt.Printf("read from filer %s: %v\n", filerAddress, err)
 		return false
 	}
 	if strings.HasPrefix(urlPath, dirBuckets+"/") {
@@ -174,9 +159,8 @@ func runCopy(cmd *Command, args []string) bool {
 		go func() {
 			defer waitGroup.Done()
 			worker := FileCopyWorker{
-				options:          &copy,
-				filerHost:        filerUrl.Host,
-				filerGrpcAddress: filerGrpcAddress,
+				options:      &copy,
+				filerAddress: filerAddress,
 			}
 			if err := worker.copyFiles(fileCopyTaskChan); err != nil {
 				fmt.Fprintf(os.Stderr, "copy file error: %v\n", err)
@@ -189,7 +173,7 @@ func runCopy(cmd *Command, args []string) bool {
 	return true
 }
 
-func readFilerConfiguration(grpcDialOption grpc.DialOption, filerGrpcAddress string) (masters []string, collection, replication string, dirBuckets string, maxMB uint32, cipher bool, err error) {
+func readFilerConfiguration(grpcDialOption grpc.DialOption, filerGrpcAddress pb.ServerAddress) (masters []string, collection, replication string, dirBuckets string, maxMB uint32, cipher bool, err error) {
 	err = pb.WithGrpcFilerClient(filerGrpcAddress, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
 		resp, err := client.GetFilerConfiguration(context.Background(), &filer_pb.GetFilerConfigurationRequest{})
 		if err != nil {
@@ -241,9 +225,8 @@ func genFileCopyTask(fileOrDir string, destPath string, fileCopyTaskChan chan Fi
 }
 
 type FileCopyWorker struct {
-	options          *CopyOptions
-	filerHost        string
-	filerGrpcAddress string
+	options      *CopyOptions
+	filerAddress pb.ServerAddress
 }
 
 func (worker *FileCopyWorker) copyFiles(fileCopyTaskChan chan FileCopyTask) error {
@@ -321,7 +304,7 @@ func (worker *FileCopyWorker) checkExistingFileFirst(task FileCopyTask, f *os.Fi
 		return
 	}
 
-	err = pb.WithGrpcFilerClient(worker.filerGrpcAddress, worker.options.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
+	err = pb.WithGrpcFilerClient(worker.filerAddress, worker.options.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
 
 		request := &filer_pb.LookupDirectoryEntryRequest{
 			Directory: task.destinationUrlPath,
@@ -361,9 +344,9 @@ func (worker *FileCopyWorker) uploadFileAsOne(task FileCopyTask, f *os.File) err
 			return err
 		}
 
-		// assign a volume
-		err = util.Retry("assignVolume", func() error {
-			return pb.WithGrpcFilerClient(worker.filerGrpcAddress, worker.options.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
+		err = util.Retry("upload", func() error {
+			// assign a volume
+			assignErr := pb.WithGrpcFilerClient(worker.filerAddress, worker.options.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
 
 				request := &filer_pb.AssignVolumeRequest{
 					Count:       1,
@@ -381,43 +364,49 @@ func (worker *FileCopyWorker) uploadFileAsOne(task FileCopyTask, f *os.File) err
 				if assignResult.Error != "" {
 					return fmt.Errorf("assign volume failure %v: %v", request, assignResult.Error)
 				}
-				if assignResult.Url == "" {
+				if assignResult.Location.Url == "" {
 					return fmt.Errorf("assign volume failure %v: %v", request, assignResult)
 				}
 				return nil
 			})
-		})
-		if err != nil {
-			return fmt.Errorf("Failed to assign from %v: %v\n", worker.options.masters, err)
-		}
+			if assignErr != nil {
+				return assignErr
+			}
 
-		targetUrl := "http://" + assignResult.Url + "/" + assignResult.FileId
-		uploadOption := &operation.UploadOption{
-			UploadUrl:         targetUrl,
-			Filename:          fileName,
-			Cipher:            worker.options.cipher,
-			IsInputCompressed: false,
-			MimeType:          mimeType,
-			PairMap:           nil,
-			Jwt:               security.EncodedJwt(assignResult.Auth),
-		}
-		uploadResult, err := operation.UploadData(data, uploadOption)
+			// upload data
+			targetUrl := "http://" + assignResult.Location.Url + "/" + assignResult.FileId
+			uploadOption := &operation.UploadOption{
+				UploadUrl:         targetUrl,
+				Filename:          fileName,
+				Cipher:            worker.options.cipher,
+				IsInputCompressed: false,
+				MimeType:          mimeType,
+				PairMap:           nil,
+				Jwt:               security.EncodedJwt(assignResult.Auth),
+			}
+			uploadResult, err := operation.UploadData(data, uploadOption)
+			if err != nil {
+				return fmt.Errorf("upload data %v to %s: %v\n", fileName, targetUrl, err)
+			}
+			if uploadResult.Error != "" {
+				return fmt.Errorf("upload %v to %s result: %v\n", fileName, targetUrl, uploadResult.Error)
+			}
+			if *worker.options.verbose {
+				fmt.Printf("uploaded %s to %s\n", fileName, targetUrl)
+			}
+
+			fmt.Printf("copied %s => http://%s%s%s\n", f.Name(), worker.filerAddress.ToHttpAddress(), task.destinationUrlPath, fileName)
+			chunks = append(chunks, uploadResult.ToPbFileChunk(assignResult.FileId, 0))
+
+			return nil
+		})
 		if err != nil {
-			return fmt.Errorf("upload data %v to %s: %v\n", fileName, targetUrl, err)
+			return fmt.Errorf("upload %v: %v\n", fileName, err)
 		}
-		if uploadResult.Error != "" {
-			return fmt.Errorf("upload %v to %s result: %v\n", fileName, targetUrl, uploadResult.Error)
-		}
-		if *worker.options.verbose {
-			fmt.Printf("uploaded %s to %s\n", fileName, targetUrl)
-		}
-
-		chunks = append(chunks, uploadResult.ToPbFileChunk(assignResult.FileId, 0))
 
-		fmt.Printf("copied %s => http://%s%s%s\n", f.Name(), worker.filerHost, task.destinationUrlPath, fileName)
 	}
 
-	if err := pb.WithGrpcFilerClient(worker.filerGrpcAddress, worker.options.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
+	if err := pb.WithGrpcFilerClient(worker.filerAddress, worker.options.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
 		request := &filer_pb.CreateEntryRequest{
 			Directory: task.destinationUrlPath,
 			Entry: &filer_pb.Entry{
@@ -443,7 +432,7 @@ func (worker *FileCopyWorker) uploadFileAsOne(task FileCopyTask, f *os.File) err
 		}
 		return nil
 	}); err != nil {
-		return fmt.Errorf("upload data %v to http://%s%s%s: %v\n", fileName, worker.filerHost, task.destinationUrlPath, fileName, err)
+		return fmt.Errorf("upload data %v to http://%s%s%s: %v\n", fileName, worker.filerAddress.ToHttpAddress(), task.destinationUrlPath, fileName, err)
 	}
 
 	return nil
@@ -474,7 +463,7 @@ func (worker *FileCopyWorker) uploadFileInChunks(task FileCopyTask, f *os.File,
 			var assignResult *filer_pb.AssignVolumeResponse
 			var assignError error
 			err := util.Retry("assignVolume", func() error {
-				return pb.WithGrpcFilerClient(worker.filerGrpcAddress, worker.options.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
+				return pb.WithGrpcFilerClient(worker.filerAddress, worker.options.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
 					request := &filer_pb.AssignVolumeRequest{
 						Count:       1,
 						Replication: *worker.options.replication,
@@ -498,7 +487,7 @@ func (worker *FileCopyWorker) uploadFileInChunks(task FileCopyTask, f *os.File,
 				fmt.Printf("Failed to assign from %v: %v\n", worker.options.masters, err)
 			}
 
-			targetUrl := "http://" + assignResult.Url + "/" + assignResult.FileId
+			targetUrl := "http://" + assignResult.Location.Url + "/" + assignResult.FileId
 			if collection == "" {
 				collection = assignResult.Collection
 			}
@@ -508,7 +497,7 @@ func (worker *FileCopyWorker) uploadFileInChunks(task FileCopyTask, f *os.File,
 
 			uploadOption := &operation.UploadOption{
 				UploadUrl:         targetUrl,
-				Filename:          fileName+"-"+strconv.FormatInt(i+1, 10),
+				Filename:          fileName + "-" + strconv.FormatInt(i+1, 10),
 				Cipher:            worker.options.cipher,
 				IsInputCompressed: false,
 				MimeType:          "",
@@ -542,8 +531,8 @@ func (worker *FileCopyWorker) uploadFileInChunks(task FileCopyTask, f *os.File,
 		for _, chunk := range chunks {
 			fileIds = append(fileIds, chunk.FileId)
 		}
-		operation.DeleteFiles(func() string {
-			return copy.masters[0]
+		operation.DeleteFiles(func() pb.ServerAddress {
+			return pb.ServerAddress(copy.masters[0])
 		}, false, worker.options.grpcDialOption, fileIds)
 		return uploadError
 	}
@@ -553,7 +542,7 @@ func (worker *FileCopyWorker) uploadFileInChunks(task FileCopyTask, f *os.File,
 		return fmt.Errorf("create manifest: %v", manifestErr)
 	}
 
-	if err := pb.WithGrpcFilerClient(worker.filerGrpcAddress, worker.options.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
+	if err := pb.WithGrpcFilerClient(worker.filerAddress, worker.options.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
 		request := &filer_pb.CreateEntryRequest{
 			Directory: task.destinationUrlPath,
 			Entry: &filer_pb.Entry{
@@ -579,10 +568,10 @@ func (worker *FileCopyWorker) uploadFileInChunks(task FileCopyTask, f *os.File,
 		}
 		return nil
 	}); err != nil {
-		return fmt.Errorf("upload data %v to http://%s%s%s: %v\n", fileName, worker.filerHost, task.destinationUrlPath, fileName, err)
+		return fmt.Errorf("upload data %v to http://%s%s%s: %v\n", fileName, worker.filerAddress.ToHttpAddress(), task.destinationUrlPath, fileName, err)
 	}
 
-	fmt.Printf("copied %s => http://%s%s%s\n", f.Name(), worker.filerHost, task.destinationUrlPath, fileName)
+	fmt.Printf("copied %s => http://%s%s%s\n", f.Name(), worker.filerAddress.ToHttpAddress(), task.destinationUrlPath, fileName)
 
 	return nil
 }
@@ -611,7 +600,7 @@ func (worker *FileCopyWorker) saveDataAsChunk(reader io.Reader, name string, off
 	var fileId, host string
 	var auth security.EncodedJwt
 
-	if flushErr := pb.WithGrpcFilerClient(worker.filerGrpcAddress, worker.options.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
+	if flushErr := pb.WithGrpcFilerClient(worker.filerAddress, worker.options.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
 
 		ctx := context.Background()
 
@@ -633,7 +622,7 @@ func (worker *FileCopyWorker) saveDataAsChunk(reader io.Reader, name string, off
 				return fmt.Errorf("assign volume failure %v: %v", request, resp.Error)
 			}
 
-			fileId, host, auth = resp.FileId, resp.Url, security.EncodedJwt(resp.Auth)
+			fileId, host, auth = resp.FileId, resp.Location.Url, security.EncodedJwt(resp.Auth)
 			collection, replication = resp.Collection, resp.Replication
 
 			return nil

weed/command/filer_meta_backup.go

@@ -195,7 +195,7 @@ func (metaBackup *FilerMetaBackupOptions) streamMetadataBackup() error {
 		return metaBackup.setOffset(lastTime)
 	})
 
-	return pb.FollowMetadata(*metaBackup.filerAddress, metaBackup.grpcDialOption, "meta_backup",
+	return pb.FollowMetadata(pb.ServerAddress(*metaBackup.filerAddress), metaBackup.grpcDialOption, "meta_backup",
 		*metaBackup.filerDirectory, nil, startTime.UnixNano(), 0, processEventFnWithOffset, false)
 
 }
@@ -224,7 +224,7 @@ var _ = filer_pb.FilerClient(&FilerMetaBackupOptions{})
 
 func (metaBackup *FilerMetaBackupOptions) WithFilerClient(fn func(filer_pb.SeaweedFilerClient) error) error {
 
-	return pb.WithFilerClient(*metaBackup.filerAddress, metaBackup.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
+	return pb.WithFilerClient(pb.ServerAddress(*metaBackup.filerAddress), metaBackup.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
 		return fn(client)
 	})
 

weed/command/filer_meta_tail.go

@@ -71,12 +71,12 @@ func runFilerMetaTail(cmd *Command, args []string) bool {
 	}
 
 	shouldPrint := func(resp *filer_pb.SubscribeMetadataResponse) bool {
-		if filterFunc == nil {
-			return true
-		}
 		if resp.EventNotification.OldEntry == nil && resp.EventNotification.NewEntry == nil {
 			return false
 		}
+		if filterFunc == nil {
+			return true
+		}
 		if resp.EventNotification.OldEntry != nil && filterFunc(resp.Directory, resp.EventNotification.OldEntry.Name) {
 			return true
 		}
@@ -103,7 +103,7 @@ func runFilerMetaTail(cmd *Command, args []string) bool {
 		}
 	}
 
-	tailErr := pb.FollowMetadata(*tailFiler, grpcDialOption, "tail",
+	tailErr := pb.FollowMetadata(pb.ServerAddress(*tailFiler), grpcDialOption, "tail",
 		*tailTarget, nil, time.Now().Add(-*tailStart).UnixNano(), 0,
 		func(resp *filer_pb.SubscribeMetadataResponse) error {
 			if !shouldPrint(resp) {

weed/command/filer_remote_gateway.go

@@ -0,0 +1,117 @@
+package command
+
+import (
+	"context"
+	"fmt"
+	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/pb"
+	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
+	"github.com/chrislusf/seaweedfs/weed/pb/remote_pb"
+	"github.com/chrislusf/seaweedfs/weed/replication/source"
+	"github.com/chrislusf/seaweedfs/weed/security"
+	"github.com/chrislusf/seaweedfs/weed/util"
+	"google.golang.org/grpc"
+	"os"
+	"time"
+)
+
+type RemoteGatewayOptions struct {
+	filerAddress             *string
+	grpcDialOption           grpc.DialOption
+	readChunkFromFiler       *bool
+	timeAgo                  *time.Duration
+	createBucketAt           *string
+	createBucketRandomSuffix *bool
+	include                  *string
+	exclude                  *string
+
+	mappings    *remote_pb.RemoteStorageMapping
+	remoteConfs map[string]*remote_pb.RemoteConf
+	bucketsDir  string
+}
+
+var _ = filer_pb.FilerClient(&RemoteGatewayOptions{})
+
+func (option *RemoteGatewayOptions) WithFilerClient(fn func(filer_pb.SeaweedFilerClient) error) error {
+	return pb.WithFilerClient(pb.ServerAddress(*option.filerAddress), option.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
+		return fn(client)
+	})
+}
+func (option *RemoteGatewayOptions) AdjustedUrl(location *filer_pb.Location) string {
+	return location.Url
+}
+
+var (
+	remoteGatewayOptions RemoteGatewayOptions
+)
+
+func init() {
+	cmdFilerRemoteGateway.Run = runFilerRemoteGateway // break init cycle
+	remoteGatewayOptions.filerAddress = cmdFilerRemoteGateway.Flag.String("filer", "localhost:8888", "filer of the SeaweedFS cluster")
+	remoteGatewayOptions.createBucketAt = cmdFilerRemoteGateway.Flag.String("createBucketAt", "", "one remote storage name to create new buckets in")
+	remoteGatewayOptions.createBucketRandomSuffix = cmdFilerRemoteGateway.Flag.Bool("createBucketWithRandomSuffix", true, "add randomized suffix to bucket name to avoid conflicts")
+	remoteGatewayOptions.readChunkFromFiler = cmdFilerRemoteGateway.Flag.Bool("filerProxy", false, "read file chunks from filer instead of volume servers")
+	remoteGatewayOptions.timeAgo = cmdFilerRemoteGateway.Flag.Duration("timeAgo", 0, "start time before now. \"300ms\", \"1.5h\" or \"2h45m\". Valid time units are \"ns\", \"us\" (or \"µs\"), \"ms\", \"s\", \"m\", \"h\"")
+	remoteGatewayOptions.include = cmdFilerRemoteGateway.Flag.String("include", "", "pattens of new bucket names, e.g., s3*")
+	remoteGatewayOptions.exclude = cmdFilerRemoteGateway.Flag.String("exclude", "", "pattens of new bucket names, e.g., local*")
+}
+
+var cmdFilerRemoteGateway = &Command{
+	UsageLine: "filer.remote.gateway",
+	Short:     "resumable continuously write back bucket creation, deletion, and other local updates to remote object store",
+	Long: `resumable continuously write back bucket creation, deletion, and other local updates to remote object store
+
+	filer.remote.gateway listens on filer local buckets update events. 
+	If any bucket is created, deleted, or updated, it will mirror the changes to remote object store.
+
+		weed filer.remote.sync -createBucketAt=cloud1
+
+`,
+}
+
+func runFilerRemoteGateway(cmd *Command, args []string) bool {
+
+	util.LoadConfiguration("security", false)
+	grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client")
+	remoteGatewayOptions.grpcDialOption = grpcDialOption
+
+	filerAddress := pb.ServerAddress(*remoteGatewayOptions.filerAddress)
+
+	filerSource := &source.FilerSource{}
+	filerSource.DoInitialize(
+		filerAddress.ToHttpAddress(),
+		filerAddress.ToGrpcAddress(),
+		"/", // does not matter
+		*remoteGatewayOptions.readChunkFromFiler,
+	)
+
+	remoteGatewayOptions.bucketsDir = "/buckets"
+	// check buckets again
+	remoteGatewayOptions.WithFilerClient(func(filerClient filer_pb.SeaweedFilerClient) error {
+		resp, err := filerClient.GetFilerConfiguration(context.Background(), &filer_pb.GetFilerConfigurationRequest{})
+		if err != nil {
+			return err
+		}
+		remoteGatewayOptions.bucketsDir = resp.DirBuckets
+		return nil
+	})
+
+	// read filer remote storage mount mappings
+	if detectErr := remoteGatewayOptions.collectRemoteStorageConf(); detectErr != nil {
+		fmt.Fprintf(os.Stderr, "read mount info: %v\n", detectErr)
+		return true
+	}
+
+	// synchronize /buckets folder
+	fmt.Printf("synchronize buckets in %s ...\n", remoteGatewayOptions.bucketsDir)
+	util.RetryForever("filer.remote.sync buckets", func() error {
+		return remoteGatewayOptions.followBucketUpdatesAndUploadToRemote(filerSource)
+	}, func(err error) bool {
+		if err != nil {
+			glog.Errorf("synchronize %s: %v", remoteGatewayOptions.bucketsDir, err)
+		}
+		return true
+	})
+	return true
+
+}

weed/command/filer_remote_sync_buckets.go → weed/command/filer_remote_gateway_buckets.go

@@ -13,11 +13,12 @@ import (
 	"github.com/golang/protobuf/proto"
 	"math"
 	"math/rand"
+	"path/filepath"
 	"strings"
 	"time"
 )
 
-func (option *RemoteSyncOptions) followBucketUpdatesAndUploadToRemote(filerSource *source.FilerSource) error {
+func (option *RemoteGatewayOptions) followBucketUpdatesAndUploadToRemote(filerSource *source.FilerSource) error {
 
 	// read filer remote storage mount mappings
 	if detectErr := option.collectRemoteStorageConf(); detectErr != nil {
@@ -32,16 +33,16 @@ func (option *RemoteSyncOptions) followBucketUpdatesAndUploadToRemote(filerSourc
 	processEventFnWithOffset := pb.AddOffsetFunc(eachEntryFunc, 3*time.Second, func(counter int64, lastTsNs int64) error {
 		lastTime := time.Unix(0, lastTsNs)
 		glog.V(0).Infof("remote sync %s progressed to %v %0.2f/sec", *option.filerAddress, lastTime, float64(counter)/float64(3))
-		return remote_storage.SetSyncOffset(option.grpcDialOption, *option.filerAddress, option.bucketsDir, lastTsNs)
+		return remote_storage.SetSyncOffset(option.grpcDialOption, pb.ServerAddress(*option.filerAddress), option.bucketsDir, lastTsNs)
 	})
 
-	lastOffsetTs := collectLastSyncOffset(option, option.bucketsDir)
+	lastOffsetTs := collectLastSyncOffset(option, option.grpcDialOption, pb.ServerAddress(*option.filerAddress), option.bucketsDir, *option.timeAgo)
 
-	return pb.FollowMetadata(*option.filerAddress, option.grpcDialOption, "filer.remote.sync",
+	return pb.FollowMetadata(pb.ServerAddress(*option.filerAddress), option.grpcDialOption, "filer.remote.sync",
 		option.bucketsDir, []string{filer.DirectoryEtcRemote}, lastOffsetTs.UnixNano(), 0, processEventFnWithOffset, false)
 }
 
-func (option *RemoteSyncOptions) makeBucketedEventProcessor(filerSource *source.FilerSource) (pb.ProcessMetadataFunc, error) {
+func (option *RemoteGatewayOptions) makeBucketedEventProcessor(filerSource *source.FilerSource) (pb.ProcessMetadataFunc, error) {
 
 	handleCreateBucket := func(entry *filer_pb.Entry) error {
 		if !entry.IsDirectory {
@@ -75,6 +76,16 @@ func (option *RemoteSyncOptions) makeBucketedEventProcessor(filerSource *source.
 		}
 
 		bucketName := strings.ToLower(entry.Name)
+		if *option.include != "" {
+			if ok, _ := filepath.Match(*option.include, entry.Name); !ok {
+				return nil
+			}
+		}
+		if *option.exclude != "" {
+			if ok, _ := filepath.Match(*option.exclude, entry.Name); ok {
+				return nil
+			}
+		}
 		if *option.createBucketRandomSuffix {
 			// https://docs.aws.amazon.com/AmazonS3/latest/userguide/bucketnamingrules.html
 			if len(bucketName)+5 > 63 {
@@ -307,7 +318,7 @@ func (option *RemoteSyncOptions) makeBucketedEventProcessor(filerSource *source.
 	return eachEntryFunc, nil
 }
 
-func (option *RemoteSyncOptions) findRemoteStorageClient(bucketName string) (client remote_storage.RemoteStorageClient, remoteStorageMountLocation *remote_pb.RemoteStorageLocation, err error) {
+func (option *RemoteGatewayOptions) findRemoteStorageClient(bucketName string) (client remote_storage.RemoteStorageClient, remoteStorageMountLocation *remote_pb.RemoteStorageLocation, err error) {
 	bucket := util.FullPath(option.bucketsDir).Child(bucketName)
 
 	var isMounted bool
@@ -327,7 +338,7 @@ func (option *RemoteSyncOptions) findRemoteStorageClient(bucketName string) (cli
 	return client, remoteStorageMountLocation, nil
 }
 
-func (option *RemoteSyncOptions) detectBucketInfo(actualDir string) (bucket util.FullPath, remoteStorageMountLocation *remote_pb.RemoteStorageLocation, remoteConf *remote_pb.RemoteConf, ok bool) {
+func (option *RemoteGatewayOptions) detectBucketInfo(actualDir string) (bucket util.FullPath, remoteStorageMountLocation *remote_pb.RemoteStorageLocation, remoteConf *remote_pb.RemoteConf, ok bool) {
 	bucket, ok = extractBucketPath(option.bucketsDir, actualDir)
 	if !ok {
 		return "", nil, nil, false
@@ -355,9 +366,9 @@ func extractBucketPath(bucketsDir, dir string) (util.FullPath, bool) {
 	return util.FullPath(bucketsDir).Child(parts[0]), true
 }
 
-func (option *RemoteSyncOptions) collectRemoteStorageConf() (err error) {
+func (option *RemoteGatewayOptions) collectRemoteStorageConf() (err error) {
 
-	if mappings, err := filer.ReadMountMappings(option.grpcDialOption, *option.filerAddress); err != nil {
+	if mappings, err := filer.ReadMountMappings(option.grpcDialOption, pb.ServerAddress(*option.filerAddress)); err != nil {
 		return err
 	} else {
 		option.mappings = mappings

weed/command/filer_remote_sync.go

@@ -1,39 +1,29 @@
 package command
 
 import (
-	"context"
 	"fmt"
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/chrislusf/seaweedfs/weed/pb"
 	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
-	"github.com/chrislusf/seaweedfs/weed/pb/remote_pb"
 	"github.com/chrislusf/seaweedfs/weed/replication/source"
 	"github.com/chrislusf/seaweedfs/weed/security"
 	"github.com/chrislusf/seaweedfs/weed/util"
 	"google.golang.org/grpc"
-	"os"
 	"time"
 )
 
 type RemoteSyncOptions struct {
-	filerAddress             *string
-	grpcDialOption           grpc.DialOption
-	readChunkFromFiler       *bool
-	debug                    *bool
-	timeAgo                  *time.Duration
-	dir                      *string
-	createBucketAt           *string
-	createBucketRandomSuffix *bool
-
-	mappings    *remote_pb.RemoteStorageMapping
-	remoteConfs map[string]*remote_pb.RemoteConf
-	bucketsDir  string
+	filerAddress       *string
+	grpcDialOption     grpc.DialOption
+	readChunkFromFiler *bool
+	timeAgo            *time.Duration
+	dir                *string
 }
 
 var _ = filer_pb.FilerClient(&RemoteSyncOptions{})
 
 func (option *RemoteSyncOptions) WithFilerClient(fn func(filer_pb.SeaweedFilerClient) error) error {
-	return pb.WithFilerClient(*option.filerAddress, option.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
+	return pb.WithFilerClient(pb.ServerAddress(*option.filerAddress), option.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
 		return fn(client)
 	})
 }
@@ -49,10 +39,7 @@ func init() {
 	cmdFilerRemoteSynchronize.Run = runFilerRemoteSynchronize // break init cycle
 	remoteSyncOptions.filerAddress = cmdFilerRemoteSynchronize.Flag.String("filer", "localhost:8888", "filer of the SeaweedFS cluster")
 	remoteSyncOptions.dir = cmdFilerRemoteSynchronize.Flag.String("dir", "", "a mounted directory on filer")
-	remoteSyncOptions.createBucketAt = cmdFilerRemoteSynchronize.Flag.String("createBucketAt", "", "one remote storage name to create new buckets in")
-	remoteSyncOptions.createBucketRandomSuffix = cmdFilerRemoteSynchronize.Flag.Bool("createBucketWithRandomSuffix", true, "add randomized suffix to bucket name to avoid conflicts")
 	remoteSyncOptions.readChunkFromFiler = cmdFilerRemoteSynchronize.Flag.Bool("filerProxy", false, "read file chunks from filer instead of volume servers")
-	remoteSyncOptions.debug = cmdFilerRemoteSynchronize.Flag.Bool("debug", false, "debug mode to print out filer updated remote files")
 	remoteSyncOptions.timeAgo = cmdFilerRemoteSynchronize.Flag.Duration("timeAgo", 0, "start time before now. \"300ms\", \"1.5h\" or \"2h45m\". Valid time units are \"ns\", \"us\" (or \"µs\"), \"ms\", \"s\", \"m\", \"h\"")
 }
 
@@ -65,21 +52,8 @@ var cmdFilerRemoteSynchronize = &Command{
 	If any mounted remote file is updated, it will fetch the updated content,
 	and write to the remote storage.
 
-	There are two modes:
-
-	1)By default, watch /buckets folder and write back all changes.
-
-		# if there is only one remote storage configured
-		weed filer.remote.sync
-		# if there are multiple remote storages configured
-		# specify a remote storage to create new buckets.
-		weed filer.remote.sync -createBucketAt=cloud1
-
-	2)Write back one mounted folder to remote storage
-
 		weed filer.remote.sync -dir=/mount/s3_on_cloud
 
-
 `,
 }
 
@@ -90,28 +64,17 @@ func runFilerRemoteSynchronize(cmd *Command, args []string) bool {
 	remoteSyncOptions.grpcDialOption = grpcDialOption
 
 	dir := *remoteSyncOptions.dir
-	filerAddress := *remoteSyncOptions.filerAddress
+	filerAddress := pb.ServerAddress(*remoteSyncOptions.filerAddress)
 
 	filerSource := &source.FilerSource{}
 	filerSource.DoInitialize(
-		filerAddress,
-		pb.ServerToGrpcAddress(filerAddress),
+		filerAddress.ToHttpAddress(),
+		filerAddress.ToGrpcAddress(),
 		"/", // does not matter
 		*remoteSyncOptions.readChunkFromFiler,
 	)
 
-	remoteSyncOptions.bucketsDir = "/buckets"
-	// check buckets again
-	remoteSyncOptions.WithFilerClient(func(filerClient filer_pb.SeaweedFilerClient) error {
-		resp, err := filerClient.GetFilerConfiguration(context.Background(), &filer_pb.GetFilerConfigurationRequest{})
-		if err != nil {
-			return err
-		}
-		remoteSyncOptions.bucketsDir = resp.DirBuckets
-		return nil
-	})
-
-	if dir != "" && dir != remoteSyncOptions.bucketsDir {
+	if dir != "" {
 		fmt.Printf("synchronize %s to remote storage...\n", dir)
 		util.RetryForever("filer.remote.sync "+dir, func() error {
 			return followUpdatesAndUploadToRemote(&remoteSyncOptions, filerSource, dir)
@@ -124,22 +87,6 @@ func runFilerRemoteSynchronize(cmd *Command, args []string) bool {
 		return true
 	}
 
-	// read filer remote storage mount mappings
-	if detectErr := remoteSyncOptions.collectRemoteStorageConf(); detectErr != nil {
-		fmt.Fprintf(os.Stderr, "read mount info: %v\n", detectErr)
-		return true
-	}
-
-	// synchronize /buckets folder
-	fmt.Printf("synchronize buckets in %s ...\n", remoteSyncOptions.bucketsDir)
-	util.RetryForever("filer.remote.sync buckets", func() error {
-		return remoteSyncOptions.followBucketUpdatesAndUploadToRemote(filerSource)
-	}, func(err error) bool {
-		if err != nil {
-			glog.Errorf("synchronize %s: %v", remoteSyncOptions.bucketsDir, err)
-		}
-		return true
-	})
 	return true
 
 }

weed/command/filer_remote_sync_dir.go

@@ -12,6 +12,7 @@ import (
 	"github.com/chrislusf/seaweedfs/weed/replication/source"
 	"github.com/chrislusf/seaweedfs/weed/util"
 	"github.com/golang/protobuf/proto"
+	"google.golang.org/grpc"
 	"os"
 	"strings"
 	"time"
@@ -20,7 +21,7 @@ import (
 func followUpdatesAndUploadToRemote(option *RemoteSyncOptions, filerSource *source.FilerSource, mountedDir string) error {
 
 	// read filer remote storage mount mappings
-	_, _, remoteStorageMountLocation, remoteStorage, detectErr := filer.DetectMountInfo(option.grpcDialOption, *option.filerAddress, mountedDir)
+	_, _, remoteStorageMountLocation, remoteStorage, detectErr := filer.DetectMountInfo(option.grpcDialOption, pb.ServerAddress(*option.filerAddress), mountedDir)
 	if detectErr != nil {
 		return fmt.Errorf("read mount info: %v", detectErr)
 	}
@@ -33,12 +34,12 @@ func followUpdatesAndUploadToRemote(option *RemoteSyncOptions, filerSource *sour
 	processEventFnWithOffset := pb.AddOffsetFunc(eachEntryFunc, 3*time.Second, func(counter int64, lastTsNs int64) error {
 		lastTime := time.Unix(0, lastTsNs)
 		glog.V(0).Infof("remote sync %s progressed to %v %0.2f/sec", *option.filerAddress, lastTime, float64(counter)/float64(3))
-		return remote_storage.SetSyncOffset(option.grpcDialOption, *option.filerAddress, mountedDir, lastTsNs)
+		return remote_storage.SetSyncOffset(option.grpcDialOption, pb.ServerAddress(*option.filerAddress), mountedDir, lastTsNs)
 	})
 
-	lastOffsetTs := collectLastSyncOffset(option, mountedDir)
+	lastOffsetTs := collectLastSyncOffset(option, option.grpcDialOption, pb.ServerAddress(*option.filerAddress), mountedDir, *option.timeAgo)
 
-	return pb.FollowMetadata(*option.filerAddress, option.grpcDialOption, "filer.remote.sync",
+	return pb.FollowMetadata(pb.ServerAddress(*option.filerAddress), option.grpcDialOption, "filer.remote.sync",
 		mountedDir, []string{filer.DirectoryEtcRemote}, lastOffsetTs.UnixNano(), 0, processEventFnWithOffset, false)
 }
 
@@ -159,19 +160,19 @@ func makeEventProcessor(remoteStorage *remote_pb.RemoteConf, mountedDir string,
 	return eachEntryFunc, nil
 }
 
-func collectLastSyncOffset(option *RemoteSyncOptions, mountedDir string) time.Time {
+func collectLastSyncOffset(filerClient filer_pb.FilerClient, grpcDialOption grpc.DialOption, filerAddress pb.ServerAddress, mountedDir string, timeAgo time.Duration) time.Time {
 	// 1. specified by timeAgo
 	// 2. last offset timestamp for this directory
 	// 3. directory creation time
 	var lastOffsetTs time.Time
-	if *option.timeAgo == 0 {
-		mountedDirEntry, err := filer_pb.GetEntry(option, util.FullPath(mountedDir))
+	if timeAgo == 0 {
+		mountedDirEntry, err := filer_pb.GetEntry(filerClient, util.FullPath(mountedDir))
 		if err != nil {
 			glog.V(0).Infof("get mounted directory %s: %v", mountedDir, err)
 			return time.Now()
 		}
 
-		lastOffsetTsNs, err := remote_storage.GetSyncOffset(option.grpcDialOption, *option.filerAddress, mountedDir)
+		lastOffsetTsNs, err := remote_storage.GetSyncOffset(grpcDialOption, filerAddress, mountedDir)
 		if mountedDirEntry != nil {
 			if err == nil && mountedDirEntry.Attributes.Crtime < lastOffsetTsNs/1000000 {
 				lastOffsetTs = time.Unix(0, lastOffsetTsNs)
@@ -183,7 +184,7 @@ func collectLastSyncOffset(option *RemoteSyncOptions, mountedDir string) time.Ti
 			lastOffsetTs = time.Now()
 		}
 	} else {
-		lastOffsetTs = time.Now().Add(-*option.timeAgo)
+		lastOffsetTs = time.Now().Add(-timeAgo)
 	}
 	return lastOffsetTs
 }

weed/command/filer_sync.go

@@ -93,9 +93,11 @@ func runFilerSynchronize(cmd *Command, args []string) bool {
 
 	grace.SetupProfiling(*syncCpuProfile, *syncMemProfile)
 
+	filerA := pb.ServerAddress(*syncOptions.filerA)
+	filerB := pb.ServerAddress(*syncOptions.filerB)
 	go func() {
 		for {
-			err := doSubscribeFilerMetaChanges(grpcDialOption, *syncOptions.filerA, *syncOptions.aPath, *syncOptions.aProxyByFiler, *syncOptions.filerB,
+			err := doSubscribeFilerMetaChanges(grpcDialOption, filerA, *syncOptions.aPath, *syncOptions.aProxyByFiler, filerB,
 				*syncOptions.bPath, *syncOptions.bReplication, *syncOptions.bCollection, *syncOptions.bTtlSec, *syncOptions.bProxyByFiler, *syncOptions.bDiskType, *syncOptions.bDebug)
 			if err != nil {
 				glog.Errorf("sync from %s to %s: %v", *syncOptions.filerA, *syncOptions.filerB, err)
@@ -107,7 +109,7 @@ func runFilerSynchronize(cmd *Command, args []string) bool {
 	if !*syncOptions.isActivePassive {
 		go func() {
 			for {
-				err := doSubscribeFilerMetaChanges(grpcDialOption, *syncOptions.filerB, *syncOptions.bPath, *syncOptions.bProxyByFiler, *syncOptions.filerA,
+				err := doSubscribeFilerMetaChanges(grpcDialOption, filerB, *syncOptions.bPath, *syncOptions.bProxyByFiler, filerA,
 					*syncOptions.aPath, *syncOptions.aReplication, *syncOptions.aCollection, *syncOptions.aTtlSec, *syncOptions.aProxyByFiler, *syncOptions.aDiskType, *syncOptions.aDebug)
 				if err != nil {
 					glog.Errorf("sync from %s to %s: %v", *syncOptions.filerB, *syncOptions.filerA, err)
@@ -122,7 +124,7 @@ func runFilerSynchronize(cmd *Command, args []string) bool {
 	return true
 }
 
-func doSubscribeFilerMetaChanges(grpcDialOption grpc.DialOption, sourceFiler, sourcePath string, sourceReadChunkFromFiler bool, targetFiler, targetPath string,
+func doSubscribeFilerMetaChanges(grpcDialOption grpc.DialOption, sourceFiler pb.ServerAddress, sourcePath string, sourceReadChunkFromFiler bool, targetFiler pb.ServerAddress, targetPath string,
 	replicationStr, collection string, ttlSec int, sinkWriteChunkByFiler bool, diskType string, debug bool) error {
 
 	// read source filer signature
@@ -147,9 +149,9 @@ func doSubscribeFilerMetaChanges(grpcDialOption grpc.DialOption, sourceFiler, so
 
 	// create filer sink
 	filerSource := &source.FilerSource{}
-	filerSource.DoInitialize(sourceFiler, pb.ServerToGrpcAddress(sourceFiler), sourcePath, sourceReadChunkFromFiler)
+	filerSource.DoInitialize(sourceFiler.ToHttpAddress(), sourceFiler.ToGrpcAddress(), sourcePath, sourceReadChunkFromFiler)
 	filerSink := &filersink.FilerSink{}
-	filerSink.DoInitialize(targetFiler, pb.ServerToGrpcAddress(targetFiler), targetPath, replicationStr, collection, ttlSec, diskType, grpcDialOption, sinkWriteChunkByFiler)
+	filerSink.DoInitialize(targetFiler.ToHttpAddress(), targetFiler.ToGrpcAddress(), targetPath, replicationStr, collection, ttlSec, diskType, grpcDialOption, sinkWriteChunkByFiler)
 	filerSink.SetSourceFiler(filerSource)
 
 	persistEventFn := genProcessFunction(sourcePath, targetPath, filerSink, debug)
@@ -170,7 +172,7 @@ func doSubscribeFilerMetaChanges(grpcDialOption grpc.DialOption, sourceFiler, so
 		return setOffset(grpcDialOption, targetFiler, SyncKeyPrefix, sourceFilerSignature, lastTsNs)
 	})
 
-	return pb.FollowMetadata(sourceFiler, grpcDialOption, "syncTo_"+targetFiler,
+	return pb.FollowMetadata(sourceFiler, grpcDialOption, "syncTo_"+string(targetFiler),
 		sourcePath, nil, sourceFilerOffsetTsNs, targetFilerSignature, processEventFnWithOffset, false)
 
 }
@@ -179,7 +181,7 @@ const (
 	SyncKeyPrefix = "sync."
 )
 
-func getOffset(grpcDialOption grpc.DialOption, filer string, signaturePrefix string, signature int32) (lastOffsetTsNs int64, readErr error) {
+func getOffset(grpcDialOption grpc.DialOption, filer pb.ServerAddress, signaturePrefix string, signature int32) (lastOffsetTsNs int64, readErr error) {
 
 	readErr = pb.WithFilerClient(filer, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
 		syncKey := []byte(signaturePrefix + "____")
@@ -206,7 +208,7 @@ func getOffset(grpcDialOption grpc.DialOption, filer string, signaturePrefix str
 
 }
 
-func setOffset(grpcDialOption grpc.DialOption, filer string, signaturePrefix string, signature int32, offsetTsNs int64) error {
+func setOffset(grpcDialOption grpc.DialOption, filer pb.ServerAddress, signaturePrefix string, signature int32, offsetTsNs int64) error {
 	return pb.WithFilerClient(filer, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
 
 		syncKey := []byte(signaturePrefix + "____")

weed/command/iam.go

@@ -43,38 +43,35 @@ func runIam(cmd *Command, args []string) bool {
 }
 
 func (iamopt *IamOptions) startIamServer() bool {
-	filerGrpcAddress, err := pb.ParseServerToGrpcAddress(*iamopt.filer)
-	if err != nil {
-		glog.Fatal(err)
-		return false
-	}
+	filerAddress := pb.ServerAddress(*iamopt.filer)
 
 	util.LoadConfiguration("security", false)
 	grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client")
 	for {
-		err = pb.WithGrpcFilerClient(filerGrpcAddress, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
+		err := pb.WithGrpcFilerClient(filerAddress, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
 			resp, err := client.GetFilerConfiguration(context.Background(), &filer_pb.GetFilerConfigurationRequest{})
 			if err != nil {
-				return fmt.Errorf("get filer %s configuration: %v", filerGrpcAddress, err)
+				return fmt.Errorf("get filer %s configuration: %v", filerAddress, err)
 			}
 			glog.V(0).Infof("IAM read filer configuration: %s", resp)
 			return nil
 		})
 		if err != nil {
-			glog.V(0).Infof("wait to connect to filer %s grpc address %s", *iamopt.filer, filerGrpcAddress)
+			glog.V(0).Infof("wait to connect to filer %s grpc address %s", *iamopt.filer, filerAddress.ToGrpcAddress())
 			time.Sleep(time.Second)
 		} else {
-			glog.V(0).Infof("connected to filer %s grpc address %s", *iamopt.filer, filerGrpcAddress)
+			glog.V(0).Infof("connected to filer %s grpc address %s", *iamopt.filer, filerAddress.ToGrpcAddress())
 			break
 		}
 	}
 
+	masters := pb.ServerAddresses(*iamopt.masters).ToAddresses()
 	router := mux.NewRouter().SkipClean(true)
 	_, iamApiServer_err := iamapi.NewIamApiServer(router, &iamapi.IamServerOption{
-		Filer:            *iamopt.filer,
-		Port:             *iamopt.port,
-		FilerGrpcAddress: filerGrpcAddress,
-		GrpcDialOption:   grpcDialOption,
+		Masters:        masters,
+		Filer:          filerAddress,
+		Port:           *iamopt.port,
+		GrpcDialOption: grpcDialOption,
 	})
 	glog.V(0).Info("NewIamApiServer created")
 	if iamApiServer_err != nil {

weed/command/imports.go

@@ -29,6 +29,6 @@ import (
 	_ "github.com/chrislusf/seaweedfs/weed/filer/postgres2"
 	_ "github.com/chrislusf/seaweedfs/weed/filer/redis"
 	_ "github.com/chrislusf/seaweedfs/weed/filer/redis2"
+	_ "github.com/chrislusf/seaweedfs/weed/filer/redis3"
 	_ "github.com/chrislusf/seaweedfs/weed/filer/sqlite"
-	_ "github.com/chrislusf/seaweedfs/weed/filer/tikv"
 )

weed/command/master.go

@@ -27,6 +27,7 @@ var (
 
 type MasterOptions struct {
 	port              *int
+	portGrpc          *int
 	ip                *string
 	ipBind            *string
 	metaFolder        *string
@@ -46,6 +47,7 @@ type MasterOptions struct {
 func init() {
 	cmdMaster.Run = runMaster // break init cycle
 	m.port = cmdMaster.Flag.Int("port", 9333, "http listen port")
+	m.portGrpc = cmdMaster.Flag.Int("port.grpc", 0, "grpc listen port")
 	m.ip = cmdMaster.Flag.String("ip", util.DetectedHostAddress(), "master <ip>|<server> address, also used as identifier")
 	m.ipBind = cmdMaster.Flag.String("ip.bind", "", "ip address to bind to")
 	m.metaFolder = cmdMaster.Flag.String("mdir", os.TempDir(), "data directory to store meta data")
@@ -111,7 +113,11 @@ func startMaster(masterOption MasterOptions, masterWhiteList []string) {
 
 	backend.LoadConfiguration(util.GetViper())
 
-	myMasterAddress, peers := checkPeers(*masterOption.ip, *masterOption.port, *masterOption.peers)
+	if *masterOption.portGrpc == 0 {
+		*masterOption.portGrpc = 10000 + *masterOption.port
+	}
+
+	myMasterAddress, peers := checkPeers(*masterOption.ip, *masterOption.port, *masterOption.portGrpc, *masterOption.peers)
 
 	r := mux.NewRouter()
 	ms := weed_server.NewMasterServer(r, masterOption.toMasterOption(masterWhiteList), peers)
@@ -130,7 +136,7 @@ func startMaster(masterOption MasterOptions, masterWhiteList []string) {
 	ms.SetRaftServer(raftServer)
 	r.HandleFunc("/cluster/status", raftServer.StatusHandler).Methods("GET")
 	// starting grpc server
-	grpcPort := *masterOption.port + 10000
+	grpcPort := *masterOption.portGrpc
 	grpcL, err := util.NewListener(util.JoinHostPort(*masterOption.ipBind, grpcPort), 0)
 	if err != nil {
 		glog.Fatalf("master failed to listen on grpc port %d: %v", grpcPort, err)
@@ -160,16 +166,14 @@ func startMaster(masterOption MasterOptions, masterWhiteList []string) {
 	select {}
 }
 
-func checkPeers(masterIp string, masterPort int, peers string) (masterAddress string, cleanedPeers []string) {
+func checkPeers(masterIp string, masterPort int, masterGrpcPort int, peers string) (masterAddress pb.ServerAddress, cleanedPeers []pb.ServerAddress) {
 	glog.V(0).Infof("current: %s:%d peers:%s", masterIp, masterPort, peers)
-	masterAddress = util.JoinHostPort(masterIp, masterPort)
-	if peers != "" {
-		cleanedPeers = strings.Split(peers, ",")
-	}
+	masterAddress = pb.NewServerAddress(masterIp, masterPort, masterGrpcPort)
+	cleanedPeers = pb.ServerAddresses(peers).ToAddresses()
 
 	hasSelf := false
 	for _, peer := range cleanedPeers {
-		if peer == masterAddress {
+		if peer.ToHttpAddress() == masterAddress.ToHttpAddress() {
 			hasSelf = true
 			break
 		}
@@ -179,13 +183,15 @@ func checkPeers(masterIp string, masterPort int, peers string) (masterAddress st
 		cleanedPeers = append(cleanedPeers, masterAddress)
 	}
 	if len(cleanedPeers)%2 == 0 {
-		glog.Fatalf("Only odd number of masters are supported!")
+		glog.Fatalf("Only odd number of masters are supported: %+v", cleanedPeers)
 	}
 	return
 }
 
-func isTheFirstOne(self string, peers []string) bool {
-	sort.Strings(peers)
+func isTheFirstOne(self pb.ServerAddress, peers []pb.ServerAddress) bool {
+	sort.Slice(peers, func(i, j int) bool {
+		return strings.Compare(string(peers[i]), string(peers[j])) < 0
+	})
 	if len(peers) <= 0 {
 		return true
 	}
@@ -193,9 +199,9 @@ func isTheFirstOne(self string, peers []string) bool {
 }
 
 func (m *MasterOptions) toMasterOption(whiteList []string) *weed_server.MasterOption {
+	masterAddress := pb.NewServerAddress(*m.ip, *m.port, *m.portGrpc)
 	return &weed_server.MasterOption{
-		Host:              *m.ip,
-		Port:              *m.port,
+		Master:            masterAddress,
 		MetaFolder:        *m.metaFolder,
 		VolumeSizeLimitMB: uint32(*m.volumeSizeLimitMB),
 		VolumePreallocate: *m.volumePreallocate,

weed/command/master_follower.go

@@ -13,7 +13,6 @@ import (
 	"github.com/gorilla/mux"
 	"google.golang.org/grpc/reflection"
 	"net/http"
-	"strings"
 	"time"
 )
 
@@ -24,6 +23,7 @@ var (
 func init() {
 	cmdMasterFollower.Run = runMasterFollower // break init cycle
 	mf.port = cmdMasterFollower.Flag.Int("port", 9334, "http listen port")
+	mf.portGrpc = cmdMasterFollower.Flag.Int("port.grpc", 0, "grpc listen port")
 	mf.ipBind = cmdMasterFollower.Flag.String("ip.bind", "", "ip address to bind to")
 	mf.peers = cmdMasterFollower.Flag.String("masters", "localhost:9333", "all master nodes in comma separated ip:port list, example: 127.0.0.1:9093,127.0.0.1:9094,127.0.0.1:9095")
 
@@ -70,6 +70,10 @@ func runMasterFollower(cmd *Command, args []string) bool {
 	util.LoadConfiguration("security", false)
 	util.LoadConfiguration("master", false)
 
+	if *mf.portGrpc == 0 {
+		*mf.portGrpc = 10000 + *mf.port
+	}
+
 	startMasterFollower(mf)
 
 	return true
@@ -78,19 +82,15 @@ func runMasterFollower(cmd *Command, args []string) bool {
 func startMasterFollower(masterOptions MasterOptions) {
 
 	// collect settings from main masters
-	masters := strings.Split(*mf.peers, ",")
-	masterGrpcAddresses, err := pb.ParseServersToGrpcAddresses(masters)
-	if err != nil {
-		glog.V(0).Infof("ParseFilerGrpcAddress: %v", err)
-		return
-	}
+	masters := pb.ServerAddresses(*mf.peers).ToAddresses()
 
+	var err error
 	grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.master")
 	for i := 0; i < 10; i++ {
-		err = pb.WithOneOfGrpcMasterClients(masterGrpcAddresses, grpcDialOption, func(client master_pb.SeaweedClient) error {
+		err = pb.WithOneOfGrpcMasterClients(masters, grpcDialOption, func(client master_pb.SeaweedClient) error {
 			resp, err := client.GetMasterConfiguration(context.Background(), &master_pb.GetMasterConfigurationRequest{})
 			if err != nil {
-				return fmt.Errorf("get master grpc address %v configuration: %v", masterGrpcAddresses, err)
+				return fmt.Errorf("get master grpc address %v configuration: %v", masters, err)
 			}
 			masterOptions.defaultReplication = &resp.DefaultReplication
 			masterOptions.volumeSizeLimitMB = aws.Uint(uint(resp.VolumeSizeLimitMB))
@@ -98,13 +98,13 @@ func startMasterFollower(masterOptions MasterOptions) {
 			return nil
 		})
 		if err != nil {
-			glog.V(0).Infof("failed to talk to filer %v: %v", masterGrpcAddresses, err)
+			glog.V(0).Infof("failed to talk to filer %v: %v", masters, err)
 			glog.V(0).Infof("wait for %d seconds ...", i+1)
 			time.Sleep(time.Duration(i+1) * time.Second)
 		}
 	}
 	if err != nil {
-		glog.Errorf("failed to talk to filer %v: %v", masterGrpcAddresses, err)
+		glog.Errorf("failed to talk to filer %v: %v", masters, err)
 		return
 	}
 
@@ -121,7 +121,7 @@ func startMasterFollower(masterOptions MasterOptions) {
 	}
 
 	// starting grpc server
-	grpcPort := *masterOptions.port + 10000
+	grpcPort := *masterOptions.portGrpc
 	grpcL, err := util.NewListener(util.JoinHostPort(*masterOptions.ipBind, grpcPort), 0)
 	if err != nil {
 		glog.Fatalf("master failed to listen on grpc port %d: %v", grpcPort, err)

weed/command/mount_std.go

@@ -70,35 +70,30 @@ func getParentInode(mountDir string) (uint64, error) {
 
 func RunMount(option *MountOptions, umask os.FileMode) bool {
 
-	filers := strings.Split(*option.filer, ",")
-	// parse filer grpc address
-	filerGrpcAddresses, err := pb.ParseServersToGrpcAddresses(filers)
-	if err != nil {
-		glog.V(0).Infof("ParseFilerGrpcAddress: %v", err)
-		return true
-	}
+	filerAddresses := pb.ServerAddresses(*option.filer).ToAddresses()
 
 	util.LoadConfiguration("security", false)
 	// try to connect to filer, filerBucketsPath may be useful later
 	grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client")
 	var cipher bool
+	var err error
 	for i := 0; i < 10; i++ {
-		err = pb.WithOneOfGrpcFilerClients(filerGrpcAddresses, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
+		err = pb.WithOneOfGrpcFilerClients(filerAddresses, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
 			resp, err := client.GetFilerConfiguration(context.Background(), &filer_pb.GetFilerConfigurationRequest{})
 			if err != nil {
-				return fmt.Errorf("get filer grpc address %v configuration: %v", filerGrpcAddresses, err)
+				return fmt.Errorf("get filer grpc address %v configuration: %v", filerAddresses, err)
 			}
 			cipher = resp.Cipher
 			return nil
 		})
 		if err != nil {
-			glog.V(0).Infof("failed to talk to filer %v: %v", filerGrpcAddresses, err)
+			glog.V(0).Infof("failed to talk to filer %v: %v", filerAddresses, err)
 			glog.V(0).Infof("wait for %d seconds ...", i+1)
 			time.Sleep(time.Duration(i+1) * time.Second)
 		}
 	}
 	if err != nil {
-		glog.Errorf("failed to talk to filer %v: %v", filerGrpcAddresses, err)
+		glog.Errorf("failed to talk to filer %v: %v", filerAddresses, err)
 		return true
 	}
 
@@ -206,8 +201,7 @@ func RunMount(option *MountOptions, umask os.FileMode) bool {
 
 	seaweedFileSystem := filesys.NewSeaweedFileSystem(&filesys.Option{
 		MountDirectory:     dir,
-		FilerAddresses:     filers,
-		FilerGrpcAddresses: filerGrpcAddresses,
+		FilerAddresses:     filerAddresses,
 		GrpcDialOption:     grpcDialOption,
 		FilerMountRootPath: mountRoot,
 		Collection:         *option.collection,

weed/command/msg_broker.go

@@ -62,35 +62,31 @@ func (msgBrokerOpt *MessageBrokerOptions) startQueueServer() bool {
 
 	grace.SetupProfiling(*messageBrokerStandaloneOptions.cpuprofile, *messageBrokerStandaloneOptions.memprofile)
 
-	filerGrpcAddress, err := pb.ParseServerToGrpcAddress(*msgBrokerOpt.filer)
-	if err != nil {
-		glog.Fatal(err)
-		return false
-	}
+	filerAddress := pb.ServerAddress(*msgBrokerOpt.filer)
 
 	grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.msg_broker")
 	cipher := false
 
 	for {
-		err = pb.WithGrpcFilerClient(filerGrpcAddress, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
+		err := pb.WithGrpcFilerClient(filerAddress, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
 			resp, err := client.GetFilerConfiguration(context.Background(), &filer_pb.GetFilerConfigurationRequest{})
 			if err != nil {
-				return fmt.Errorf("get filer %s configuration: %v", filerGrpcAddress, err)
+				return fmt.Errorf("get filer %s configuration: %v", filerAddress, err)
 			}
 			cipher = resp.Cipher
 			return nil
 		})
 		if err != nil {
-			glog.V(0).Infof("wait to connect to filer %s grpc address %s", *msgBrokerOpt.filer, filerGrpcAddress)
+			glog.V(0).Infof("wait to connect to filer %s grpc address %s", *msgBrokerOpt.filer, filerAddress.ToGrpcAddress())
 			time.Sleep(time.Second)
 		} else {
-			glog.V(0).Infof("connected to filer %s grpc address %s", *msgBrokerOpt.filer, filerGrpcAddress)
+			glog.V(0).Infof("connected to filer %s grpc address %s", *msgBrokerOpt.filer, filerAddress.ToGrpcAddress())
 			break
 		}
 	}
 
 	qs, err := broker.NewMessageBroker(&broker.MessageBrokerOption{
-		Filers:             []string{*msgBrokerOpt.filer},
+		Filers:             []pb.ServerAddress{filerAddress},
 		DefaultReplication: "",
 		MaxMB:              0,
 		Ip:                 *msgBrokerOpt.ip,

weed/command/s3.go

@@ -42,7 +42,7 @@ func init() {
 	s3StandaloneOptions.tlsPrivateKey = cmdS3.Flag.String("key.file", "", "path to the TLS private key file")
 	s3StandaloneOptions.tlsCertificate = cmdS3.Flag.String("cert.file", "", "path to the TLS certificate file")
 	s3StandaloneOptions.metricsHttpPort = cmdS3.Flag.Int("metricsPort", 0, "Prometheus metrics listen port")
-	s3StandaloneOptions.allowEmptyFolder = cmdS3.Flag.Bool("allowEmptyFolder", false, "allow empty folders")
+	s3StandaloneOptions.allowEmptyFolder = cmdS3.Flag.Bool("allowEmptyFolder", true, "allow empty folders")
 }
 
 var cmdS3 = &Command{
@@ -137,11 +137,7 @@ func runS3(cmd *Command, args []string) bool {
 
 func (s3opt *S3Options) startS3Server() bool {
 
-	filerGrpcAddress, err := pb.ParseServerToGrpcAddress(*s3opt.filer)
-	if err != nil {
-		glog.Fatal(err)
-		return false
-	}
+	filerAddress := pb.ServerAddress(*s3opt.filer)
 
 	filerBucketsPath := "/buckets"
 
@@ -152,10 +148,10 @@ func (s3opt *S3Options) startS3Server() bool {
 	var metricsIntervalSec int
 
 	for {
-		err = pb.WithGrpcFilerClient(filerGrpcAddress, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
+		err := pb.WithGrpcFilerClient(filerAddress, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
 			resp, err := client.GetFilerConfiguration(context.Background(), &filer_pb.GetFilerConfigurationRequest{})
 			if err != nil {
-				return fmt.Errorf("get filer %s configuration: %v", filerGrpcAddress, err)
+				return fmt.Errorf("get filer %s configuration: %v", filerAddress, err)
 			}
 			filerBucketsPath = resp.DirBuckets
 			metricsAddress, metricsIntervalSec = resp.MetricsAddress, int(resp.MetricsIntervalSec)
@@ -163,10 +159,10 @@ func (s3opt *S3Options) startS3Server() bool {
 			return nil
 		})
 		if err != nil {
-			glog.V(0).Infof("wait to connect to filer %s grpc address %s", *s3opt.filer, filerGrpcAddress)
+			glog.V(0).Infof("wait to connect to filer %s grpc address %s", *s3opt.filer, filerAddress.ToGrpcAddress())
 			time.Sleep(time.Second)
 		} else {
-			glog.V(0).Infof("connected to filer %s grpc address %s", *s3opt.filer, filerGrpcAddress)
+			glog.V(0).Infof("connected to filer %s grpc address %s", *s3opt.filer, filerAddress.ToGrpcAddress())
 			break
 		}
 	}
@@ -176,9 +172,8 @@ func (s3opt *S3Options) startS3Server() bool {
 	router := mux.NewRouter().SkipClean(true)
 
 	_, s3ApiServer_err := s3api.NewS3ApiServer(router, &s3api.S3ApiServerOption{
-		Filer:            *s3opt.filer,
+		Filer:            filerAddress,
 		Port:             *s3opt.port,
-		FilerGrpcAddress: filerGrpcAddress,
 		Config:           *s3opt.config,
 		DomainName:       *s3opt.domainName,
 		BucketsPath:      filerBucketsPath,

weed/command/scaffold/filer.toml

@@ -185,6 +185,28 @@ routeByLatency = false
 # This changes the data layout. Only add new directories. Removing/Updating will cause data loss.
 superLargeDirectories = []
 
+[redis3] # beta
+enabled = false
+address = "localhost:6379"
+password = ""
+database = 0
+
+[redis_cluster3] # beta
+enabled = false
+addresses = [
+    "localhost:30001",
+    "localhost:30002",
+    "localhost:30003",
+    "localhost:30004",
+    "localhost:30005",
+    "localhost:30006",
+]
+password = ""
+# allows reads from slave servers or the master, but all writes still go to the master
+readOnly = false
+# automatically use the closest Redis server for reads
+routeByLatency = false
+
 [etcd]
 enabled = false
 servers = "localhost:2379"
@@ -230,11 +252,3 @@ location = "/tmp/"
 address = "localhost:6379"
 password = ""
 database = 1
-
-[tikv]
-enabled = false
-# If you have many pd address, use ',' split then:
-#   pdaddrs = "pdhost1:2379, pdhost2:2379, pdhost3:2379"
-pdaddrs = "localhost:2379"
-# Concurrency for TiKV delete range
-deleterange_concurrency = 1

weed/command/scaffold/master.toml

@@ -23,10 +23,7 @@ default = "localhost:8888"    # used by maintenance scripts if the scripts needs
 
 
 [master.sequencer]
-type = "raft"     # Choose [raft|etcd|snowflake] type for storing the file id sequence
-# when sequencer.type = etcd, set listen client urls of etcd cluster that store file id sequence
-# example : http://127.0.0.1:2379,http://127.0.0.1:2389
-sequencer_etcd_urls = "http://127.0.0.1:2379"
+type = "raft"     # Choose [raft|snowflake] type for storing the file id sequence
 # when sequencer.type = snowflake, the snowflake id must be different from other masters
 sequencer_snowflake_id = 0     # any number between 1~1023
 

weed/command/server.go

@@ -2,6 +2,7 @@ package command
 
 import (
 	"fmt"
+	"github.com/chrislusf/seaweedfs/weed/pb"
 	"github.com/chrislusf/seaweedfs/weed/util/grace"
 	"net/http"
 	"os"
@@ -85,6 +86,7 @@ func init() {
 	serverOptions.debugPort = cmdServer.Flag.Int("debug.port", 6060, "http port for debugging")
 
 	masterOptions.port = cmdServer.Flag.Int("master.port", 9333, "master server http listen port")
+	masterOptions.portGrpc = cmdServer.Flag.Int("master.port.grpc", 0, "master server grpc listen port")
 	masterOptions.metaFolder = cmdServer.Flag.String("master.dir", "", "data directory to store meta data, default to same as -dir specified")
 	masterOptions.peers = cmdServer.Flag.String("master.peers", "", "all master nodes in comma separated ip:masterPort list")
 	masterOptions.volumeSizeLimitMB = cmdServer.Flag.Uint("master.volumeSizeLimitMB", 30*1000, "Master stops directing writes to oversized volumes.")
@@ -97,6 +99,7 @@ func init() {
 
 	filerOptions.collection = cmdServer.Flag.String("filer.collection", "", "all data will be stored in this collection")
 	filerOptions.port = cmdServer.Flag.Int("filer.port", 8888, "filer server http listen port")
+	filerOptions.portGrpc = cmdServer.Flag.Int("filer.port.grpc", 0, "filer server grpc listen port")
 	filerOptions.publicPort = cmdServer.Flag.Int("filer.port.public", 0, "filer server public http listen port")
 	filerOptions.defaultReplicaPlacement = cmdServer.Flag.String("filer.defaultReplicaPlacement", "", "default replication type. If not specified, use master setting.")
 	filerOptions.disableDirListing = cmdServer.Flag.Bool("filer.disableDirListing", false, "turn off directory listing")
@@ -108,6 +111,7 @@ func init() {
 	filerOptions.concurrentUploadLimitMB = cmdServer.Flag.Int("filer.concurrentUploadLimitMB", 64, "limit total concurrent upload size")
 
 	serverOptions.v.port = cmdServer.Flag.Int("volume.port", 8080, "volume server http listen port")
+	serverOptions.v.portGrpc = cmdServer.Flag.Int("volume.port.grpc", 0, "volume server grpc listen port")
 	serverOptions.v.publicPort = cmdServer.Flag.Int("volume.port.public", 0, "volume server public port")
 	serverOptions.v.indexType = cmdServer.Flag.String("volume.index", "memory", "Choose [memory|leveldb|leveldbMedium|leveldbLarge] mode for memory~performance balance.")
 	serverOptions.v.diskType = cmdServer.Flag.String("volume.disk", "", "[hdd|ssd|<tag>] hard drive or solid state drive or any tag")
@@ -128,7 +132,7 @@ func init() {
 	s3Options.tlsPrivateKey = cmdServer.Flag.String("s3.key.file", "", "path to the TLS private key file")
 	s3Options.tlsCertificate = cmdServer.Flag.String("s3.cert.file", "", "path to the TLS certificate file")
 	s3Options.config = cmdServer.Flag.String("s3.config", "", "path to the config file")
-	s3Options.allowEmptyFolder = cmdServer.Flag.Bool("s3.allowEmptyFolder", false, "allow empty folders")
+	s3Options.allowEmptyFolder = cmdServer.Flag.Bool("s3.allowEmptyFolder", true, "allow empty folders")
 
 	webdavOptions.port = cmdServer.Flag.Int("webdav.port", 7333, "webdav server http listen port")
 	webdavOptions.collection = cmdServer.Flag.String("webdav.collection", "", "collection to create the files")
@@ -165,20 +169,20 @@ func runServer(cmd *Command, args []string) bool {
 	}
 
 	if *isStartingMasterServer {
-		_, peerList := checkPeers(*serverIp, *masterOptions.port, *masterOptions.peers)
-		peers := strings.Join(peerList, ",")
+		_, peerList := checkPeers(*serverIp, *masterOptions.port, *masterOptions.portGrpc, *masterOptions.peers)
+		peers := strings.Join(pb.ToAddressStrings(peerList), ",")
 		masterOptions.peers = &peers
 	}
 
 	// ip address
 	masterOptions.ip = serverIp
 	masterOptions.ipBind = serverBindIp
-	filerOptions.masters = masterOptions.peers
+	filerOptions.masters = pb.ServerAddresses(*masterOptions.peers).ToAddresses()
 	filerOptions.ip = serverIp
 	filerOptions.bindIp = serverBindIp
 	serverOptions.v.ip = serverIp
 	serverOptions.v.bindIp = serverBindIp
-	serverOptions.v.masters = masterOptions.peers
+	serverOptions.v.masters = pb.ServerAddresses(*masterOptions.peers).ToAddresses()
 	serverOptions.v.idleConnectionTimeout = serverTimeout
 	serverOptions.v.dataCenter = serverDataCenter
 	serverOptions.v.rack = serverRack
@@ -194,7 +198,7 @@ func runServer(cmd *Command, args []string) bool {
 	filerOptions.disableHttp = serverDisableHttp
 	masterOptions.disableHttp = serverDisableHttp
 
-	filerAddress := util.JoinHostPort(*serverIp, *filerOptions.port)
+	filerAddress := string(pb.NewServerAddress(*serverIp, *filerOptions.port, *filerOptions.portGrpc))
 	s3Options.filer = &filerAddress
 	webdavOptions.filer = &filerAddress
 	msgBrokerOptions.filer = &filerAddress

weed/command/shell.go

@@ -2,6 +2,7 @@ package command
 
 import (
 	"fmt"
+	"github.com/chrislusf/seaweedfs/weed/pb"
 
 	"github.com/chrislusf/seaweedfs/weed/security"
 	"github.com/chrislusf/seaweedfs/weed/shell"
@@ -53,13 +54,7 @@ func runShell(command *Command, args []string) bool {
 
 	fmt.Printf("master: %s filer: %s\n", *shellOptions.Masters, *shellInitialFiler)
 
-	var err error
-	shellOptions.FilerHost, shellOptions.FilerPort, err = util.ParseHostPort(*shellInitialFiler)
-	shellOptions.FilerAddress = *shellInitialFiler
-	if err != nil {
-		fmt.Printf("failed to parse filer %s: %v\n", *shellInitialFiler, err)
-		return false
-	}
+	shellOptions.FilerAddress = pb.ServerAddress(*shellInitialFiler)
 	shellOptions.Directory = "/"
 
 	shell.RunShell(shellOptions)

weed/command/upload.go

@@ -71,7 +71,7 @@ func runUpload(cmd *Command, args []string) bool {
 	util.LoadConfiguration("security", false)
 	grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client")
 
-	defaultReplication, err := readMasterConfiguration(grpcDialOption, *upload.master)
+	defaultReplication, err := readMasterConfiguration(grpcDialOption, pb.ServerAddress(*upload.master))
 	if err != nil {
 		fmt.Printf("upload: %v", err)
 		return false
@@ -96,7 +96,7 @@ func runUpload(cmd *Command, args []string) bool {
 					if e != nil {
 						return e
 					}
-					results, e := operation.SubmitFiles(func() string { return *upload.master }, grpcDialOption, parts, *upload.replication, *upload.collection, *upload.dataCenter, *upload.ttl, *upload.diskType, *upload.maxMB, *upload.usePublicUrl)
+					results, e := operation.SubmitFiles(func() pb.ServerAddress { return pb.ServerAddress(*upload.master) }, grpcDialOption, parts, *upload.replication, *upload.collection, *upload.dataCenter, *upload.ttl, *upload.diskType, *upload.maxMB, *upload.usePublicUrl)
 					bytes, _ := json.Marshal(results)
 					fmt.Println(string(bytes))
 					if e != nil {
@@ -118,7 +118,7 @@ func runUpload(cmd *Command, args []string) bool {
 			fmt.Println(e.Error())
 			return false
 		}
-		results, err := operation.SubmitFiles(func() string { return *upload.master }, grpcDialOption, parts, *upload.replication, *upload.collection, *upload.dataCenter, *upload.ttl, *upload.diskType, *upload.maxMB, *upload.usePublicUrl)
+		results, err := operation.SubmitFiles(func() pb.ServerAddress { return pb.ServerAddress(*upload.master) }, grpcDialOption, parts, *upload.replication, *upload.collection, *upload.dataCenter, *upload.ttl, *upload.diskType, *upload.maxMB, *upload.usePublicUrl)
 		if err != nil {
 			fmt.Println(err.Error())
 			return false
@@ -129,7 +129,7 @@ func runUpload(cmd *Command, args []string) bool {
 	return true
 }
 
-func readMasterConfiguration(grpcDialOption grpc.DialOption, masterAddress string) (replication string, err error) {
+func readMasterConfiguration(grpcDialOption grpc.DialOption, masterAddress pb.ServerAddress) (replication string, err error) {
 	err = pb.WithMasterClient(masterAddress, grpcDialOption, func(client master_pb.SeaweedClient) error {
 		resp, err := client.GetMasterConfiguration(context.Background(), &master_pb.GetMasterConfigurationRequest{})
 		if err != nil {

weed/command/volume.go

@@ -36,6 +36,7 @@ var (
 
 type VolumeServerOptions struct {
 	port                      *int
+	portGrpc                  *int
 	publicPort                *int
 	folders                   []string
 	folderMaxLimits           []int
@@ -43,7 +44,8 @@ type VolumeServerOptions struct {
 	ip                        *string
 	publicUrl                 *string
 	bindIp                    *string
-	masters                   *string
+	mastersString             *string
+	masters                   []pb.ServerAddress
 	idleConnectionTimeout     *int
 	dataCenter                *string
 	rack                      *string
@@ -68,11 +70,12 @@ type VolumeServerOptions struct {
 func init() {
 	cmdVolume.Run = runVolume // break init cycle
 	v.port = cmdVolume.Flag.Int("port", 8080, "http listen port")
+	v.portGrpc = cmdVolume.Flag.Int("port.grpc", 0, "grpc listen port")
 	v.publicPort = cmdVolume.Flag.Int("port.public", 0, "port opened to public")
 	v.ip = cmdVolume.Flag.String("ip", util.DetectedHostAddress(), "ip or server name, also used as identifier")
 	v.publicUrl = cmdVolume.Flag.String("publicUrl", "", "Publicly accessible address")
 	v.bindIp = cmdVolume.Flag.String("ip.bind", "", "ip address to bind to")
-	v.masters = cmdVolume.Flag.String("mserver", "localhost:9333", "comma-separated master servers")
+	v.mastersString = cmdVolume.Flag.String("mserver", "localhost:9333", "comma-separated master servers")
 	v.preStopSeconds = cmdVolume.Flag.Int("preStopSeconds", 10, "number of seconds between stop send heartbeats and stop volume server")
 	// v.pulseSeconds = cmdVolume.Flag.Int("pulseSeconds", 5, "number of seconds between heartbeats, must be smaller than or equal to the master's setting")
 	v.idleConnectionTimeout = cmdVolume.Flag.Int("idleTimeout", 30, "connection idle seconds")
@@ -123,6 +126,7 @@ func runVolume(cmd *Command, args []string) bool {
 	go stats_collect.StartMetricsServer(*v.metricsHttpPort)
 
 	minFreeSpaces := util.MustParseMinFreeSpace(*minFreeSpace, *minFreeSpacePercent)
+	v.masters = pb.ServerAddresses(*v.mastersString).ToAddresses()
 	v.startVolumeServer(*volumeFolders, *maxVolumeCounts, *volumeWhiteListOption, minFreeSpaces)
 
 	return true
@@ -193,6 +197,9 @@ func (v VolumeServerOptions) startVolumeServer(volumeFolders, maxVolumeCounts, v
 	if *v.publicPort == 0 {
 		*v.publicPort = *v.port
 	}
+	if *v.portGrpc == 0 {
+		*v.portGrpc = 10000 + *v.port
+	}
 	if *v.publicUrl == "" {
 		*v.publicUrl = util.JoinHostPort(*v.ip, *v.publicPort)
 	}
@@ -221,14 +228,12 @@ func (v VolumeServerOptions) startVolumeServer(volumeFolders, maxVolumeCounts, v
 		volumeNeedleMapKind = storage.NeedleMapLevelDbLarge
 	}
 
-	masters := *v.masters
-
 	volumeServer := weed_server.NewVolumeServer(volumeMux, publicVolumeMux,
-		*v.ip, *v.port, *v.publicUrl,
+		*v.ip, *v.port, *v.portGrpc, *v.publicUrl,
 		v.folders, v.folderMaxLimits, minFreeSpaces, diskTypes,
 		*v.idxFolder,
 		volumeNeedleMapKind,
-		strings.Split(masters, ","), 5, *v.dataCenter, *v.rack,
+		v.masters, 5, *v.dataCenter, *v.rack,
 		v.whiteList,
 		*v.fixJpgOrientation, *v.readMode,
 		*v.compactionMBPerSecond,
@@ -307,7 +312,7 @@ func (v VolumeServerOptions) isSeparatedPublicPort() bool {
 }
 
 func (v VolumeServerOptions) startGrpcService(vs volume_server_pb.VolumeServerServer) *grpc.Server {
-	grpcPort := *v.port + 10000
+	grpcPort := *v.portGrpc
 	grpcL, err := util.NewListener(util.JoinHostPort(*v.bindIp, grpcPort), 0)
 	if err != nil {
 		glog.Fatalf("failed to listen on grpc port %d: %v", grpcPort, err)
@@ -373,7 +378,7 @@ func (v VolumeServerOptions) startClusterHttpService(handler http.Handler) httpd
 }
 
 func (v VolumeServerOptions) startTcpService(volumeServer *weed_server.VolumeServer) {
-	listeningAddress := util.JoinHostPort(*v.bindIp,*v.port+20000)
+	listeningAddress := util.JoinHostPort(*v.bindIp, *v.port+20000)
 	glog.V(0).Infoln("Start Seaweed volume server", util.Version(), "tcp at", listeningAddress)
 	listener, e := util.NewListener(listeningAddress, 0)
 	if e != nil {

weed/command/webdav.go

@@ -78,46 +78,41 @@ func (wo *WebDavOption) startWebDav() bool {
 	}
 
 	// parse filer grpc address
-	filerGrpcAddress, err := pb.ParseServerToGrpcAddress(*wo.filer)
-	if err != nil {
-		glog.Fatal(err)
-		return false
-	}
+	filerAddress := pb.ServerAddress(*wo.filer)
 
 	grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.client")
 
 	var cipher bool
 	// connect to filer
 	for {
-		err = pb.WithGrpcFilerClient(filerGrpcAddress, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
+		err := pb.WithGrpcFilerClient(filerAddress, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
 			resp, err := client.GetFilerConfiguration(context.Background(), &filer_pb.GetFilerConfigurationRequest{})
 			if err != nil {
-				return fmt.Errorf("get filer %s configuration: %v", filerGrpcAddress, err)
+				return fmt.Errorf("get filer %s configuration: %v", filerAddress, err)
 			}
 			cipher = resp.Cipher
 			return nil
 		})
 		if err != nil {
-			glog.V(0).Infof("wait to connect to filer %s grpc address %s", *wo.filer, filerGrpcAddress)
+			glog.V(0).Infof("wait to connect to filer %s grpc address %s", *wo.filer, filerAddress.ToGrpcAddress())
 			time.Sleep(time.Second)
 		} else {
-			glog.V(0).Infof("connected to filer %s grpc address %s", *wo.filer, filerGrpcAddress)
+			glog.V(0).Infof("connected to filer %s grpc address %s", *wo.filer, filerAddress.ToGrpcAddress())
 			break
 		}
 	}
 
 	ws, webdavServer_err := weed_server.NewWebDavServer(&weed_server.WebDavOption{
-		Filer:            *wo.filer,
-		FilerGrpcAddress: filerGrpcAddress,
-		GrpcDialOption:   grpcDialOption,
-		Collection:       *wo.collection,
-		Replication:      *wo.replication,
-		DiskType:         *wo.disk,
-		Uid:              uid,
-		Gid:              gid,
-		Cipher:           cipher,
-		CacheDir:         util.ResolvePath(*wo.cacheDir),
-		CacheSizeMB:      *wo.cacheSizeMB,
+		Filer:          filerAddress,
+		GrpcDialOption: grpcDialOption,
+		Collection:     *wo.collection,
+		Replication:    *wo.replication,
+		DiskType:       *wo.disk,
+		Uid:            uid,
+		Gid:            gid,
+		Cipher:         cipher,
+		CacheDir:       util.ResolvePath(*wo.cacheDir),
+		CacheSizeMB:    *wo.cacheSizeMB,
 	})
 	if webdavServer_err != nil {
 		glog.Fatalf("WebDav Server startup error: %v", webdavServer_err)

weed/filer.toml

@@ -0,0 +1,5 @@
+[redis3]
+enabled = true
+address = "localhost:6379"
+password = ""
+database = 0

weed/filer/filer.go

@@ -3,6 +3,7 @@ package filer
 import (
 	"context"
 	"fmt"
+	"github.com/chrislusf/seaweedfs/weed/pb"
 	"os"
 	"strings"
 	"time"
@@ -43,16 +44,18 @@ type Filer struct {
 	Signature           int32
 	FilerConf           *FilerConf
 	RemoteStorage       *FilerRemoteStorage
+	UniqueFileId        uint32
 }
 
-func NewFiler(masters []string, grpcDialOption grpc.DialOption,
-	filerHost string, filerGrpcPort uint32, collection string, replication string, dataCenter string, notifyFn func()) *Filer {
+func NewFiler(masters []pb.ServerAddress, grpcDialOption grpc.DialOption,
+	filerHost pb.ServerAddress, collection string, replication string, dataCenter string, notifyFn func()) *Filer {
 	f := &Filer{
-		MasterClient:        wdclient.NewMasterClient(grpcDialOption, "filer", filerHost, filerGrpcPort, dataCenter, masters),
+		MasterClient:        wdclient.NewMasterClient(grpcDialOption, "filer", filerHost, dataCenter, masters),
 		fileIdDeletionQueue: util.NewUnboundedQueue(),
 		GrpcDialOption:      grpcDialOption,
 		FilerConf:           NewFilerConf(),
 		RemoteStorage:       NewFilerRemoteStorage(),
+		UniqueFileId:        uint32(util.RandomInt32()),
 	}
 	f.LocalMetaLogBuffer = log_buffer.NewLogBuffer("local", LogFlushInterval, f.logFlushFunc, notifyFn)
 	f.metaLogCollection = collection
@@ -63,7 +66,7 @@ func NewFiler(masters []string, grpcDialOption grpc.DialOption,
 	return f
 }
 
-func (f *Filer) AggregateFromPeers(self string, filers []string) {
+func (f *Filer) AggregateFromPeers(self pb.ServerAddress, filers []pb.ServerAddress) {
 
 	// set peers
 	found := false
@@ -110,7 +113,7 @@ func (f *Filer) GetStore() (store FilerStore) {
 	return f.Store
 }
 
-func (fs *Filer) GetMaster() string {
+func (fs *Filer) GetMaster() pb.ServerAddress {
 	return fs.MasterClient.GetMaster()
 }
 

weed/filer/filer_notify.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"io"
+	"math"
 	"strings"
 	"time"
 
@@ -92,8 +93,8 @@ func (f *Filer) logFlushFunc(startTime, stopTime time.Time, buf []byte) {
 
 	startTime, stopTime = startTime.UTC(), stopTime.UTC()
 
-	targetFile := fmt.Sprintf("%s/%04d-%02d-%02d/%02d-%02d.segment", SystemLogDir,
-		startTime.Year(), startTime.Month(), startTime.Day(), startTime.Hour(), startTime.Minute(),
+	targetFile := fmt.Sprintf("%s/%04d-%02d-%02d/%02d-%02d.%08x", SystemLogDir,
+		startTime.Year(), startTime.Month(), startTime.Day(), startTime.Hour(), startTime.Minute(), f.UniqueFileId,
 		// startTime.Second(), startTime.Nanosecond(),
 	)
 
@@ -111,7 +112,7 @@ func (f *Filer) ReadPersistedLogBuffer(startTime time.Time, eachLogEntryFn func(
 
 	startTime = startTime.UTC()
 	startDate := fmt.Sprintf("%04d-%02d-%02d", startTime.Year(), startTime.Month(), startTime.Day())
-	startHourMinute := fmt.Sprintf("%02d-%02d.segment", startTime.Hour(), startTime.Minute())
+	startHourMinute := fmt.Sprintf("%02d-%02d", startTime.Hour(), startTime.Minute())
 
 	sizeBuf := make([]byte, 4)
 	startTsNs := startTime.UnixNano()
@@ -122,14 +123,15 @@ func (f *Filer) ReadPersistedLogBuffer(startTime time.Time, eachLogEntryFn func(
 	}
 	for _, dayEntry := range dayEntries {
 		// println("checking day", dayEntry.FullPath)
-		hourMinuteEntries, _, listHourMinuteErr := f.ListDirectoryEntries(context.Background(), util.NewFullPath(SystemLogDir, dayEntry.Name()), "", false, 24*60, "", "", "")
+		hourMinuteEntries, _, listHourMinuteErr := f.ListDirectoryEntries(context.Background(), util.NewFullPath(SystemLogDir, dayEntry.Name()), "", false, math.MaxInt32, "", "", "")
 		if listHourMinuteErr != nil {
 			return lastTsNs, fmt.Errorf("fail to list log %s by day: %v", dayEntry.Name(), listHourMinuteErr)
 		}
 		for _, hourMinuteEntry := range hourMinuteEntries {
 			// println("checking hh-mm", hourMinuteEntry.FullPath)
 			if dayEntry.Name() == startDate {
-				if strings.Compare(hourMinuteEntry.Name(), startHourMinute) < 0 {
+				hourMinute := util.FileNameBase(hourMinuteEntry.Name())
+				if strings.Compare(hourMinute, startHourMinute) < 0 {
 					continue
 				}
 			}

weed/filer/filer_notify_append.go

@@ -33,6 +33,8 @@ func (f *Filer) appendToFile(targetFile string, data []byte) error {
 				Gid:    OS_GID,
 			},
 		}
+	} else if err != nil {
+		return fmt.Errorf("find %s: %v", fullpath, err)
 	} else {
 		offset = int64(TotalSize(entry.Chunks))
 	}

weed/filer/filer_search.go

@@ -23,15 +23,15 @@ func splitPattern(pattern string) (prefix string, restPattern string) {
 // For now, prefix and namePattern are mutually exclusive
 func (f *Filer) ListDirectoryEntries(ctx context.Context, p util.FullPath, startFileName string, inclusive bool, limit int64, prefix string, namePattern string, namePatternExclude string) (entries []*Entry, hasMore bool, err error) {
 
+	if limit > math.MaxInt32-1 {
+		limit = math.MaxInt32 - 1
+	}
+
 	_, err = f.StreamListDirectoryEntries(ctx, p, startFileName, inclusive, limit+1, prefix, namePattern, namePatternExclude, func(entry *Entry) bool {
 		entries = append(entries, entry)
 		return true
 	})
 
-	if limit == math.MaxInt64 {
-		limit = math.MaxInt64 - 1
-	}
-
 	hasMore = int64(len(entries)) >= limit+1
 	if hasMore {
 		entries = entries[:limit]

weed/filer/filerstore_translate_path.go

@@ -3,6 +3,7 @@ package filer
 import (
 	"context"
 	"github.com/chrislusf/seaweedfs/weed/util"
+	"math"
 	"strings"
 )
 
@@ -120,6 +121,10 @@ func (t *FilerStorePathTranlator) ListDirectoryPrefixedEntries(ctx context.Conte
 
 	newFullPath := t.translatePath(dirPath)
 
+	if limit > math.MaxInt32-1 {
+		limit = math.MaxInt32 - 1
+	}
+
 	return t.actualStore.ListDirectoryPrefixedEntries(ctx, newFullPath, startFileName, includeStartFile, limit, prefix, func(entry *Entry) bool {
 		entry.FullPath = dirPath[:len(t.storeRoot)-1] + entry.FullPath
 		return eachEntryFunc(entry)

weed/filer/filerstore_wrapper.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"github.com/chrislusf/seaweedfs/weed/glog"
 	"github.com/viant/ptrie"
+	"math"
 	"strings"
 	"time"
 
@@ -248,6 +249,9 @@ func (fsw *FilerStoreWrapper) ListDirectoryPrefixedEntries(ctx context.Context,
 	defer func() {
 		stats.FilerStoreHistogram.WithLabelValues(actualStore.GetName(), "prefixList").Observe(time.Since(start).Seconds())
 	}()
+	if limit > math.MaxInt32-1 {
+		limit = math.MaxInt32 - 1
+	}
 	glog.V(4).Infof("ListDirectoryPrefixedEntries %s from %s prefix %s limit %d", dirPath, startFileName, prefix, limit)
 	lastFileName, err = actualStore.ListDirectoryPrefixedEntries(ctx, dirPath, startFileName, includeStartFile, limit, prefix, eachEntryFunc)
 	if err == ErrUnsupportedListDirectoryPrefixed {

weed/filer/leveldb/leveldb_store_test.go

@@ -13,7 +13,7 @@ import (
 )
 
 func TestCreateAndFind(t *testing.T) {
-	testFiler := filer.NewFiler(nil, nil, "", 0, "", "", "", nil)
+	testFiler := filer.NewFiler(nil, nil, "", "", "", "", nil)
 	dir, _ := ioutil.TempDir("", "seaweedfs_filer_test")
 	defer os.RemoveAll(dir)
 	store := &LevelDBStore{}
@@ -67,7 +67,7 @@ func TestCreateAndFind(t *testing.T) {
 }
 
 func TestEmptyRoot(t *testing.T) {
-	testFiler := filer.NewFiler(nil, nil, "", 0, "", "", "", nil)
+	testFiler := filer.NewFiler(nil, nil, "", "", "", "", nil)
 	dir, _ := ioutil.TempDir("", "seaweedfs_filer_test2")
 	defer os.RemoveAll(dir)
 	store := &LevelDBStore{}
@@ -90,7 +90,7 @@ func TestEmptyRoot(t *testing.T) {
 }
 
 func BenchmarkInsertEntry(b *testing.B) {
-	testFiler := filer.NewFiler(nil, nil, "", 0, "", "", "", nil)
+	testFiler := filer.NewFiler(nil, nil, "", "", "", "", nil)
 	dir, _ := ioutil.TempDir("", "seaweedfs_filer_bench")
 	defer os.RemoveAll(dir)
 	store := &LevelDBStore{}

weed/filer/leveldb2/leveldb2_store_test.go

@@ -11,7 +11,7 @@ import (
 )
 
 func TestCreateAndFind(t *testing.T) {
-	testFiler := filer.NewFiler(nil, nil, "", 0, "", "", "", nil)
+	testFiler := filer.NewFiler(nil, nil, "", "", "", "", nil)
 	dir, _ := ioutil.TempDir("", "seaweedfs_filer_test")
 	defer os.RemoveAll(dir)
 	store := &LevelDB2Store{}
@@ -65,7 +65,7 @@ func TestCreateAndFind(t *testing.T) {
 }
 
 func TestEmptyRoot(t *testing.T) {
-	testFiler := filer.NewFiler(nil, nil, "", 0, "", "", "", nil)
+	testFiler := filer.NewFiler(nil, nil, "", "", "", "", nil)
 	dir, _ := ioutil.TempDir("", "seaweedfs_filer_test2")
 	defer os.RemoveAll(dir)
 	store := &LevelDB2Store{}

weed/filer/leveldb3/leveldb3_store_test.go

@@ -11,7 +11,7 @@ import (
 )
 
 func TestCreateAndFind(t *testing.T) {
-	testFiler := filer.NewFiler(nil, nil, "", 0, "", "", "", nil)
+	testFiler := filer.NewFiler(nil, nil, "", "", "", "", nil)
 	dir, _ := ioutil.TempDir("", "seaweedfs_filer_test")
 	defer os.RemoveAll(dir)
 	store := &LevelDB3Store{}
@@ -65,7 +65,7 @@ func TestCreateAndFind(t *testing.T) {
 }
 
 func TestEmptyRoot(t *testing.T) {
-	testFiler := filer.NewFiler(nil, nil, "", 0, "", "", "", nil)
+	testFiler := filer.NewFiler(nil, nil, "", "", "", "", nil)
 	dir, _ := ioutil.TempDir("", "seaweedfs_filer_test2")
 	defer os.RemoveAll(dir)
 	store := &LevelDB3Store{}

weed/filer/meta_aggregator.go

@@ -18,7 +18,7 @@ import (
 )
 
 type MetaAggregator struct {
-	filers         []string
+	filers         []pb.ServerAddress
 	grpcDialOption grpc.DialOption
 	MetaLogBuffer  *log_buffer.LogBuffer
 	// notifying clients
@@ -28,7 +28,7 @@ type MetaAggregator struct {
 
 // MetaAggregator only aggregates data "on the fly". The logs are not re-persisted to disk.
 // The old data comes from what each LocalMetadata persisted on disk.
-func NewMetaAggregator(filers []string, grpcDialOption grpc.DialOption) *MetaAggregator {
+func NewMetaAggregator(filers []pb.ServerAddress, grpcDialOption grpc.DialOption) *MetaAggregator {
 	t := &MetaAggregator{
 		filers:         filers,
 		grpcDialOption: grpcDialOption,
@@ -40,13 +40,13 @@ func NewMetaAggregator(filers []string, grpcDialOption grpc.DialOption) *MetaAgg
 	return t
 }
 
-func (ma *MetaAggregator) StartLoopSubscribe(f *Filer, self string) {
+func (ma *MetaAggregator) StartLoopSubscribe(f *Filer, self pb.ServerAddress) {
 	for _, filer := range ma.filers {
 		go ma.subscribeToOneFiler(f, self, filer)
 	}
 }
 
-func (ma *MetaAggregator) subscribeToOneFiler(f *Filer, self string, peer string) {
+func (ma *MetaAggregator) subscribeToOneFiler(f *Filer, self pb.ServerAddress, peer pb.ServerAddress) {
 
 	/*
 		Each filer reads the "filer.store.id", which is the store's signature when filer starts.
@@ -110,7 +110,7 @@ func (ma *MetaAggregator) subscribeToOneFiler(f *Filer, self string, peer string
 		}
 		dir := event.Directory
 		// println("received meta change", dir, "size", len(data))
-		ma.MetaLogBuffer.AddToBuffer([]byte(dir), data, 0)
+		ma.MetaLogBuffer.AddToBuffer([]byte(dir), data, event.TsNs)
 		if maybeReplicateMetadataChange != nil {
 			maybeReplicateMetadataChange(event)
 		}
@@ -123,7 +123,7 @@ func (ma *MetaAggregator) subscribeToOneFiler(f *Filer, self string, peer string
 			ctx, cancel := context.WithCancel(context.Background())
 			defer cancel()
 			stream, err := client.SubscribeLocalMetadata(ctx, &filer_pb.SubscribeMetadataRequest{
-				ClientName: "filer:" + self,
+				ClientName: "filer:" + string(self),
 				PathPrefix: "/",
 				SinceNs:    lastTsNs,
 			})
@@ -156,7 +156,7 @@ func (ma *MetaAggregator) subscribeToOneFiler(f *Filer, self string, peer string
 	}
 }
 
-func (ma *MetaAggregator) readFilerStoreSignature(peer string) (sig int32, err error) {
+func (ma *MetaAggregator) readFilerStoreSignature(peer pb.ServerAddress) (sig int32, err error) {
 	err = pb.WithFilerClient(peer, ma.grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
 		resp, err := client.GetFilerConfiguration(context.Background(), &filer_pb.GetFilerConfigurationRequest{})
 		if err != nil {
@@ -172,7 +172,7 @@ const (
 	MetaOffsetPrefix = "Meta"
 )
 
-func (ma *MetaAggregator) readOffset(f *Filer, peer string, peerSignature int32) (lastTsNs int64, err error) {
+func (ma *MetaAggregator) readOffset(f *Filer, peer pb.ServerAddress, peerSignature int32) (lastTsNs int64, err error) {
 
 	key := []byte(MetaOffsetPrefix + "xxxx")
 	util.Uint32toBytes(key[len(MetaOffsetPrefix):], uint32(peerSignature))
@@ -195,7 +195,7 @@ func (ma *MetaAggregator) readOffset(f *Filer, peer string, peerSignature int32)
 	return
 }
 
-func (ma *MetaAggregator) updateOffset(f *Filer, peer string, peerSignature int32, lastTsNs int64) (err error) {
+func (ma *MetaAggregator) updateOffset(f *Filer, peer pb.ServerAddress, peerSignature int32, lastTsNs int64) (err error) {
 
 	key := []byte(MetaOffsetPrefix + "xxxx")
 	util.Uint32toBytes(key[len(MetaOffsetPrefix):], uint32(peerSignature))

weed/filer/redis/universal_redis_store.go

@@ -120,6 +120,8 @@ func (store *UniversalRedisStore) DeleteFolderChildren(ctx context.Context, full
 		if err != nil {
 			return fmt.Errorf("delete %s in parent dir: %v", fullpath, err)
 		}
+		// not efficient, but need to remove if it is a directory
+		store.Client.Del(ctx, genDirectoryListKey(string(path)))
 	}
 
 	return nil

weed/filer/redis2/universal_redis_store.go

@@ -144,6 +144,8 @@ func (store *UniversalRedis2Store) DeleteFolderChildren(ctx context.Context, ful
 		if err != nil {
 			return fmt.Errorf("DeleteFolderChildren %s in parent dir: %v", fullpath, err)
 		}
+		// not efficient, but need to remove if it is a directory
+		store.Client.Del(ctx, genDirectoryListKey(string(path)))
 	}
 
 	return nil

weed/filer/redis3/ItemList.go

@@ -0,0 +1,507 @@
+package redis3
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"github.com/chrislusf/seaweedfs/weed/util/skiplist"
+	"github.com/go-redis/redis/v8"
+)
+
+type ItemList struct {
+	skipList  *skiplist.SkipList
+	batchSize int
+	client    redis.UniversalClient
+	prefix    string
+}
+
+func newItemList(client redis.UniversalClient, prefix string, store skiplist.ListStore, batchSize int) *ItemList {
+	return &ItemList{
+		skipList:  skiplist.New(store),
+		batchSize: batchSize,
+		client:    client,
+		prefix:    prefix,
+	}
+}
+
+/*
+Be reluctant to create new nodes. Try to fit into either previous node or next node.
+Prefer to add to previous node.
+
+There are multiple cases after finding the name for greater or equal node
+ 1. found and node.Key == name
+    The node contains a batch with leading key the same as the name
+    nothing to do
+ 2. no such node found or node.Key > name
+
+    if no such node found
+      prevNode = list.LargestNode
+
+	// case 2.1
+    if previousNode contains name
+      nothing to do
+
+    // prefer to add to previous node
+    if prevNode != nil {
+		// case 2.2
+		if prevNode has capacity
+			prevNode.add name, and save
+			return
+		// case 2.3
+		split prevNode by name
+    }
+
+	// case 2.4
+	// merge into next node. Avoid too many nodes if adding data in reverse order.
+	if nextNode is not nil and nextNode has capacity
+	  delete nextNode.Key
+      nextNode.Key = name
+      nextNode.batch.add name
+      insert nodeNode.Key
+	  return
+
+	// case 2.5
+    if prevNode is nil
+      insert new node with key = name, value = batch{name}
+      return
+
+*/
+
+func (nl *ItemList) canAddMember(node *skiplist.SkipListElementReference, name string) (alreadyContains bool, nodeSize int, err error) {
+	ctx := context.Background()
+	pipe := nl.client.TxPipeline()
+	key := fmt.Sprintf("%s%dm", nl.prefix, node.ElementPointer)
+	countOperation := pipe.ZLexCount(ctx, key, "-", "+")
+	scoreOperationt := pipe.ZScore(ctx, key, name)
+	if _, err = pipe.Exec(ctx); err != nil && err != redis.Nil {
+		return false, 0, err
+	}
+	if err == redis.Nil {
+		err = nil
+	}
+	alreadyContains = scoreOperationt.Err() == nil
+	nodeSize = int(countOperation.Val())
+	return
+}
+
+func (nl *ItemList) WriteName(name string) error {
+
+	lookupKey := []byte(name)
+	prevNode, nextNode, found, err := nl.skipList.FindGreaterOrEqual(lookupKey)
+	if err != nil {
+		return err
+	}
+	// case 1: the name already exists as one leading key in the batch
+	if found && bytes.Compare(nextNode.Key, lookupKey) == 0 {
+		return nil
+	}
+
+	var prevNodeReference *skiplist.SkipListElementReference
+	if !found {
+		prevNodeReference = nl.skipList.GetLargestNodeReference()
+	}
+
+	if nextNode != nil && prevNode == nil {
+		prevNodeReference = nextNode.Prev
+	}
+
+	if prevNodeReference != nil {
+		alreadyContains, nodeSize, err := nl.canAddMember(prevNodeReference, name)
+		if err != nil {
+			return err
+		}
+		if alreadyContains {
+			// case 2.1
+			return nil
+		}
+
+		// case 2.2
+		if nodeSize < nl.batchSize {
+			return nl.NodeAddMember(prevNodeReference, name)
+		}
+
+		// case 2.3
+		x := nl.NodeInnerPosition(prevNodeReference, name)
+		y := nodeSize - x
+		addToX := x <= y
+		// add to a new node
+		if x == 0 || y == 0 {
+			if err := nl.ItemAdd(lookupKey, 0, name); err != nil {
+				return err
+			}
+			return nil
+		}
+		if addToX {
+			// collect names before name, add them to X
+			namesToX, err := nl.NodeRangeBeforeExclusive(prevNodeReference, name)
+			if err != nil {
+				return nil
+			}
+			// delete skiplist reference to old node
+			if _, err := nl.skipList.DeleteByKey(prevNodeReference.Key); err != nil {
+				return err
+			}
+			// add namesToY and name to a new X
+			namesToX = append(namesToX, name)
+			if err := nl.ItemAdd([]byte(namesToX[0]), 0, namesToX...); err != nil {
+				return nil
+			}
+			// remove names less than name from current Y
+			if err := nl.NodeDeleteBeforeExclusive(prevNodeReference, name); err != nil {
+				return nil
+			}
+
+			// point skip list to current Y
+			if err := nl.ItemAdd(lookupKey, prevNodeReference.ElementPointer); err != nil {
+				return nil
+			}
+			return nil
+		} else {
+			// collect names after name, add them to Y
+			namesToY, err := nl.NodeRangeAfterExclusive(prevNodeReference, name)
+			if err != nil {
+				return nil
+			}
+			// add namesToY and name to a new Y
+			namesToY = append(namesToY, name)
+			if err := nl.ItemAdd(lookupKey, 0, namesToY...); err != nil {
+				return nil
+			}
+			// remove names after name from current X
+			if err := nl.NodeDeleteAfterExclusive(prevNodeReference, name); err != nil {
+				return nil
+			}
+			return nil
+		}
+
+	}
+
+	// case 2.4
+	if nextNode != nil {
+		nodeSize := nl.NodeSize(nextNode.Reference())
+		if nodeSize < nl.batchSize {
+			if id, err := nl.skipList.DeleteByKey(nextNode.Key); err != nil {
+				return err
+			} else {
+				if err := nl.ItemAdd(lookupKey, id, name); err != nil {
+					return err
+				}
+			}
+			return nil
+		}
+	}
+
+	// case 2.5
+	// now prevNode is nil
+	return nl.ItemAdd(lookupKey, 0, name)
+}
+
+/*
+// case 1: exists in nextNode
+if nextNode != nil && nextNode.Key == name {
+	remove from nextNode, update nextNode
+	// TODO: merge with prevNode if possible?
+	return
+}
+if nextNode is nil
+	prevNode = list.Largestnode
+if prevNode == nil and nextNode.Prev != nil
+	prevNode = load(nextNode.Prev)
+
+// case 2: does not exist
+// case 2.1
+if prevNode == nil {
+	return
+}
+// case 2.2
+if prevNameBatch does not contain name {
+	return
+}
+
+// case 3
+delete from prevNameBatch
+if prevNameBatch + nextNode < capacityList
+	// case 3.1
+	merge
+else
+	// case 3.2
+	update prevNode
+
+
+*/
+func (nl *ItemList) DeleteName(name string) error {
+	lookupKey := []byte(name)
+	prevNode, nextNode, found, err := nl.skipList.FindGreaterOrEqual(lookupKey)
+	if err != nil {
+		return err
+	}
+
+	// case 1
+	if found && bytes.Compare(nextNode.Key, lookupKey) == 0 {
+		if _, err := nl.skipList.DeleteByKey(nextNode.Key); err != nil {
+			return err
+		}
+		if err := nl.NodeDeleteMember(nextNode.Reference(), name); err != nil {
+			return err
+		}
+		minName := nl.NodeMin(nextNode.Reference())
+		if minName == "" {
+			return nl.NodeDelete(nextNode.Reference())
+		}
+		return nl.ItemAdd([]byte(minName), nextNode.Id)
+	}
+
+	if !found {
+		prevNode, err = nl.skipList.GetLargestNode()
+		if err != nil {
+			return err
+		}
+	}
+
+	if nextNode != nil && prevNode == nil {
+		prevNode, err = nl.skipList.LoadElement(nextNode.Prev)
+		if err != nil {
+			return err
+		}
+	}
+
+	// case 2
+	if prevNode == nil {
+		// case 2.1
+		return nil
+	}
+	if !nl.NodeContainsItem(prevNode.Reference(), name) {
+		return nil
+	}
+
+	// case 3
+	if err := nl.NodeDeleteMember(prevNode.Reference(), name); err != nil {
+		return err
+	}
+	prevSize := nl.NodeSize(prevNode.Reference())
+	if prevSize == 0 {
+		if _, err := nl.skipList.DeleteByKey(prevNode.Key); err != nil {
+			return err
+		}
+		return nil
+	}
+	nextSize := nl.NodeSize(nextNode.Reference())
+	if nextSize > 0 && prevSize+nextSize < nl.batchSize {
+		// case 3.1 merge nextNode and prevNode
+		if _, err := nl.skipList.DeleteByKey(nextNode.Key); err != nil {
+			return err
+		}
+		nextNames, err := nl.NodeRangeBeforeExclusive(nextNode.Reference(), "")
+		if err != nil {
+			return err
+		}
+		if err := nl.NodeAddMember(prevNode.Reference(), nextNames...); err != nil {
+			return err
+		}
+		return nl.NodeDelete(nextNode.Reference())
+	} else {
+		// case 3.2 update prevNode
+		// no action to take
+		return nil
+	}
+
+	return nil
+}
+
+func (nl *ItemList) ListNames(startFrom string, visitNamesFn func(name string) bool) error {
+	lookupKey := []byte(startFrom)
+	prevNode, nextNode, found, err := nl.skipList.FindGreaterOrEqual(lookupKey)
+	if err != nil {
+		return err
+	}
+	if found && bytes.Compare(nextNode.Key, lookupKey) == 0 {
+		prevNode = nil
+	}
+	if !found {
+		prevNode, err = nl.skipList.GetLargestNode()
+		if err != nil {
+			return err
+		}
+	}
+
+	if prevNode != nil {
+		if !nl.NodeScanIncluseiveAfter(prevNode.Reference(), startFrom, visitNamesFn) {
+			return nil
+		}
+	}
+
+	for nextNode != nil {
+		if !nl.NodeScanIncluseiveAfter(nextNode.Reference(), startFrom, visitNamesFn) {
+			return nil
+		}
+		nextNode, err = nl.skipList.LoadElement(nextNode.Next[0])
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (nl *ItemList) RemoteAllListElement() error {
+
+	t := nl.skipList
+
+	nodeRef := t.StartLevels[0]
+	for nodeRef != nil {
+		node, err := t.LoadElement(nodeRef)
+		if err != nil {
+			return err
+		}
+		if node == nil {
+			return nil
+		}
+		if err := t.DeleteElement(node); err != nil {
+			return err
+		}
+		if err := nl.NodeDelete(node.Reference()); err != nil {
+			return err
+		}
+		nodeRef = node.Next[0]
+	}
+	return nil
+
+}
+
+func (nl *ItemList) NodeContainsItem(node *skiplist.SkipListElementReference, item string) bool {
+	key := fmt.Sprintf("%s%dm", nl.prefix, node.ElementPointer)
+	_, err := nl.client.ZScore(context.Background(), key, item).Result()
+	if err == redis.Nil {
+		return false
+	}
+	if err == nil {
+		return true
+	}
+	return false
+}
+
+func (nl *ItemList) NodeSize(node *skiplist.SkipListElementReference) int {
+	if node == nil {
+		return 0
+	}
+	key := fmt.Sprintf("%s%dm", nl.prefix, node.ElementPointer)
+	return int(nl.client.ZLexCount(context.Background(), key, "-", "+").Val())
+}
+
+func (nl *ItemList) NodeAddMember(node *skiplist.SkipListElementReference, names ...string) error {
+	key := fmt.Sprintf("%s%dm", nl.prefix, node.ElementPointer)
+	var members []*redis.Z
+	for _, name := range names {
+		members = append(members, &redis.Z{
+			Score:  0,
+			Member: name,
+		})
+	}
+	return nl.client.ZAddNX(context.Background(), key, members...).Err()
+}
+func (nl *ItemList) NodeDeleteMember(node *skiplist.SkipListElementReference, name string) error {
+	key := fmt.Sprintf("%s%dm", nl.prefix, node.ElementPointer)
+	return nl.client.ZRem(context.Background(), key, name).Err()
+}
+
+func (nl *ItemList) NodeDelete(node *skiplist.SkipListElementReference) error {
+	key := fmt.Sprintf("%s%dm", nl.prefix, node.ElementPointer)
+	return nl.client.Del(context.Background(), key).Err()
+}
+
+func (nl *ItemList) NodeInnerPosition(node *skiplist.SkipListElementReference, name string) int {
+	key := fmt.Sprintf("%s%dm", nl.prefix, node.ElementPointer)
+	return int(nl.client.ZLexCount(context.Background(), key, "-", "("+name).Val())
+}
+
+func (nl *ItemList) NodeMin(node *skiplist.SkipListElementReference) string {
+	key := fmt.Sprintf("%s%dm", nl.prefix, node.ElementPointer)
+	slice := nl.client.ZRangeByLex(context.Background(), key, &redis.ZRangeBy{
+		Min:    "-",
+		Max:    "+",
+		Offset: 0,
+		Count:  1,
+	}).Val()
+	if len(slice) > 0 {
+		s := slice[0]
+		return s
+	}
+	return ""
+}
+
+func (nl *ItemList) NodeScanIncluseiveAfter(node *skiplist.SkipListElementReference, startFrom string, visitNamesFn func(name string) bool) bool {
+	key := fmt.Sprintf("%s%dm", nl.prefix, node.ElementPointer)
+	if startFrom == "" {
+		startFrom = "-"
+	} else {
+		startFrom = "[" + startFrom
+	}
+	names := nl.client.ZRangeByLex(context.Background(), key, &redis.ZRangeBy{
+		Min: startFrom,
+		Max: "+",
+	}).Val()
+	for _, n := range names {
+		if !visitNamesFn(n) {
+			return false
+		}
+	}
+	return true
+}
+
+func (nl *ItemList) NodeRangeBeforeExclusive(node *skiplist.SkipListElementReference, stopAt string) ([]string, error) {
+	key := fmt.Sprintf("%s%dm", nl.prefix, node.ElementPointer)
+	if stopAt == "" {
+		stopAt = "+"
+	} else {
+		stopAt = "(" + stopAt
+	}
+	return nl.client.ZRangeByLex(context.Background(), key, &redis.ZRangeBy{
+		Min: "-",
+		Max: stopAt,
+	}).Result()
+}
+func (nl *ItemList) NodeRangeAfterExclusive(node *skiplist.SkipListElementReference, startFrom string) ([]string, error) {
+	key := fmt.Sprintf("%s%dm", nl.prefix, node.ElementPointer)
+	if startFrom == "" {
+		startFrom = "-"
+	} else {
+		startFrom = "(" + startFrom
+	}
+	return nl.client.ZRangeByLex(context.Background(), key, &redis.ZRangeBy{
+		Min: startFrom,
+		Max: "+",
+	}).Result()
+}
+
+func (nl *ItemList) NodeDeleteBeforeExclusive(node *skiplist.SkipListElementReference, stopAt string) error {
+	key := fmt.Sprintf("%s%dm", nl.prefix, node.ElementPointer)
+	if stopAt == "" {
+		stopAt = "+"
+	} else {
+		stopAt = "(" + stopAt
+	}
+	return nl.client.ZRemRangeByLex(context.Background(), key, "-", stopAt).Err()
+}
+func (nl *ItemList) NodeDeleteAfterExclusive(node *skiplist.SkipListElementReference, startFrom string) error {
+	key := fmt.Sprintf("%s%dm", nl.prefix, node.ElementPointer)
+	if startFrom == "" {
+		startFrom = "-"
+	} else {
+		startFrom = "(" + startFrom
+	}
+	return nl.client.ZRemRangeByLex(context.Background(), key, startFrom, "+").Err()
+}
+
+func (nl *ItemList) ItemAdd(lookupKey []byte, idIfKnown int64, names ...string) error {
+	if id, err := nl.skipList.InsertByKey(lookupKey, idIfKnown, nil); err != nil {
+		return err
+	} else {
+		if len(names) > 0 {
+			return nl.NodeAddMember(&skiplist.SkipListElementReference{
+				ElementPointer: id,
+				Key:            lookupKey,
+			}, names...)
+		}
+	}
+	return nil
+}

weed/filer/redis3/item_list_serde.go

@@ -0,0 +1,75 @@
+package redis3
+
+import (
+	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/util/skiplist"
+	"github.com/go-redis/redis/v8"
+	"github.com/golang/protobuf/proto"
+)
+
+func LoadItemList(data []byte, prefix string, client redis.UniversalClient, store skiplist.ListStore, batchSize int) *ItemList {
+
+	nl := &ItemList{
+		skipList:  skiplist.New(store),
+		batchSize: batchSize,
+		client:    client,
+		prefix:    prefix,
+	}
+
+	if len(data) == 0 {
+		return nl
+	}
+
+	message := &skiplist.SkipListProto{}
+	if err := proto.Unmarshal(data, message); err != nil {
+		glog.Errorf("loading skiplist: %v", err)
+	}
+	nl.skipList.MaxNewLevel = int(message.MaxNewLevel)
+	nl.skipList.MaxLevel = int(message.MaxLevel)
+	for i, ref := range message.StartLevels {
+		nl.skipList.StartLevels[i] = &skiplist.SkipListElementReference{
+			ElementPointer: ref.ElementPointer,
+			Key:            ref.Key,
+		}
+	}
+	for i, ref := range message.EndLevels {
+		nl.skipList.EndLevels[i] = &skiplist.SkipListElementReference{
+			ElementPointer: ref.ElementPointer,
+			Key:            ref.Key,
+		}
+	}
+	return nl
+}
+
+func (nl *ItemList) HasChanges() bool {
+	return nl.skipList.HasChanges
+}
+
+func (nl *ItemList) ToBytes() []byte {
+	message := &skiplist.SkipListProto{}
+	message.MaxNewLevel = int32(nl.skipList.MaxNewLevel)
+	message.MaxLevel = int32(nl.skipList.MaxLevel)
+	for _, ref := range nl.skipList.StartLevels {
+		if ref == nil {
+			break
+		}
+		message.StartLevels = append(message.StartLevels, &skiplist.SkipListElementReference{
+			ElementPointer: ref.ElementPointer,
+			Key:            ref.Key,
+		})
+	}
+	for _, ref := range nl.skipList.EndLevels {
+		if ref == nil {
+			break
+		}
+		message.EndLevels = append(message.EndLevels, &skiplist.SkipListElementReference{
+			ElementPointer: ref.ElementPointer,
+			Key:            ref.Key,
+		})
+	}
+	data, err := proto.Marshal(message)
+	if err != nil {
+		glog.Errorf("marshal skiplist: %v", err)
+	}
+	return data
+}

weed/filer/redis3/kv_directory_children.go

@@ -0,0 +1,138 @@
+package redis3
+
+import (
+	"context"
+	"fmt"
+	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/go-redis/redis/v8"
+)
+
+const maxNameBatchSizeLimit = 1000000
+
+func insertChild(ctx context.Context, redisStore *UniversalRedis3Store, key string, name string) error {
+
+	// lock and unlock
+	mutex := redisStore.redsync.NewMutex(key + "lock")
+	if err := mutex.Lock(); err != nil {
+		return fmt.Errorf("lock %s: %v", key, err)
+	}
+	defer func() {
+		mutex.Unlock()
+	}()
+
+	client := redisStore.Client
+	data, err := client.Get(ctx, key).Result()
+	if err != nil {
+		if err != redis.Nil {
+			return fmt.Errorf("read %s: %v", key, err)
+		}
+	}
+	store := newSkipListElementStore(key, client)
+	nameList := LoadItemList([]byte(data), key, client, store, maxNameBatchSizeLimit)
+
+	if err := nameList.WriteName(name); err != nil {
+		glog.Errorf("add %s %s: %v", key, name, err)
+		return err
+	}
+
+	if !nameList.HasChanges() {
+		return nil
+	}
+
+	if err := client.Set(ctx, key, nameList.ToBytes(), 0).Err(); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func removeChild(ctx context.Context, redisStore *UniversalRedis3Store, key string, name string) error {
+
+	// lock and unlock
+	mutex := redisStore.redsync.NewMutex(key + "lock")
+	if err := mutex.Lock(); err != nil {
+		return fmt.Errorf("lock %s: %v", key, err)
+	}
+	defer mutex.Unlock()
+
+	client := redisStore.Client
+	data, err := client.Get(ctx, key).Result()
+	if err != nil {
+		if err != redis.Nil {
+			return fmt.Errorf("read %s: %v", key, err)
+		}
+	}
+	store := newSkipListElementStore(key, client)
+	nameList := LoadItemList([]byte(data), key, client, store, maxNameBatchSizeLimit)
+
+	if err := nameList.DeleteName(name); err != nil {
+		return err
+	}
+	if !nameList.HasChanges() {
+		return nil
+	}
+
+	if err := client.Set(ctx, key, nameList.ToBytes(), 0).Err(); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func removeChildren(ctx context.Context, redisStore *UniversalRedis3Store, key string, onDeleteFn func(name string) error) error {
+
+	// lock and unlock
+	mutex := redisStore.redsync.NewMutex(key + "lock")
+	if err := mutex.Lock(); err != nil {
+		return fmt.Errorf("lock %s: %v", key, err)
+	}
+	defer mutex.Unlock()
+
+	client := redisStore.Client
+	data, err := client.Get(ctx, key).Result()
+	if err != nil {
+		if err != redis.Nil {
+			return fmt.Errorf("read %s: %v", key, err)
+		}
+	}
+	store := newSkipListElementStore(key, client)
+	nameList := LoadItemList([]byte(data), key, client, store, maxNameBatchSizeLimit)
+
+	if err = nameList.ListNames("", func(name string) bool {
+		if err := onDeleteFn(name); err != nil {
+			glog.Errorf("delete %s child %s: %v", key, name, err)
+			return false
+		}
+		return true
+	}); err != nil {
+		return err
+	}
+
+	if err = nameList.RemoteAllListElement(); err != nil {
+		return err
+	}
+
+	return nil
+
+}
+
+func listChildren(ctx context.Context, redisStore *UniversalRedis3Store, key string, startFileName string, eachFn func(name string) bool) error {
+	client := redisStore.Client
+	data, err := client.Get(ctx, key).Result()
+	if err != nil {
+		if err != redis.Nil {
+			return fmt.Errorf("read %s: %v", key, err)
+		}
+	}
+	store := newSkipListElementStore(key, client)
+	nameList := LoadItemList([]byte(data), key, client, store, maxNameBatchSizeLimit)
+
+	if err = nameList.ListNames(startFileName, func(name string) bool {
+		return eachFn(name)
+	}); err != nil {
+		return err
+	}
+
+	return nil
+
+}

weed/filer/redis3/kv_directory_children_test.go

@@ -0,0 +1,210 @@
+package redis3
+
+import (
+	"context"
+	"fmt"
+	"github.com/go-redis/redis/v8"
+	"github.com/stvp/tempredis"
+	"strconv"
+	"testing"
+	"time"
+)
+
+var names = []string{
+	"cassandra.in.sh",
+	"cassandra",
+	"debug-cql.bat",
+	"nodetool",
+	"nodetool.bat",
+	"source-conf.ps1",
+	"sstableloader",
+	"sstableloader.bat",
+	"sstablescrub",
+	"sstablescrub.bat",
+	"sstableupgrade",
+	"sstableupgrade.bat",
+	"sstableutil",
+	"sstableutil.bat",
+	"sstableverify",
+	"sstableverify.bat",
+	"stop-server",
+	"stop-server.bat",
+	"stop-server.ps1",
+	"cassandra.in.bat",
+	"cqlsh.py",
+	"cqlsh",
+	"cassandra.ps1",
+	"cqlsh.bat",
+	"debug-cql",
+	"cassandra.bat",
+}
+
+func yTestNameList(t *testing.T) {
+	server, err := tempredis.Start(tempredis.Config{})
+	if err != nil {
+		panic(err)
+	}
+	defer server.Term()
+
+	client := redis.NewClient(&redis.Options{
+		Network: "unix",
+		Addr:    server.Socket(),
+	})
+
+	store := newSkipListElementStore("/yyy/bin", client)
+	var data []byte
+	for _, name := range names {
+		nameList := LoadItemList(data, "/yyy/bin", client, store, maxNameBatchSizeLimit)
+		nameList.WriteName(name)
+
+		nameList.ListNames("", func(name string) bool {
+			println(name)
+			return true
+		})
+
+		if nameList.HasChanges() {
+			data = nameList.ToBytes()
+		}
+		println()
+	}
+
+	nameList := LoadItemList(data, "/yyy/bin", client, store, maxNameBatchSizeLimit)
+	nameList.ListNames("", func(name string) bool {
+		println(name)
+		return true
+	})
+
+}
+
+func yBenchmarkNameList(b *testing.B) {
+
+	server, err := tempredis.Start(tempredis.Config{})
+	if err != nil {
+		panic(err)
+	}
+	defer server.Term()
+
+	client := redis.NewClient(&redis.Options{
+		Network: "unix",
+		Addr:    server.Socket(),
+	})
+
+	store := newSkipListElementStore("/yyy/bin", client)
+	var data []byte
+	for i := 0; i < b.N; i++ {
+		nameList := LoadItemList(data, "/yyy/bin", client, store, maxNameBatchSizeLimit)
+
+		nameList.WriteName(strconv.Itoa(i) + "namexxxxxxxxxxxxxxxxxxx")
+
+		if nameList.HasChanges() {
+			data = nameList.ToBytes()
+		}
+	}
+}
+
+func BenchmarkRedis(b *testing.B) {
+
+	server, err := tempredis.Start(tempredis.Config{})
+	if err != nil {
+		panic(err)
+	}
+	defer server.Term()
+
+	client := redis.NewClient(&redis.Options{
+		Network: "unix",
+		Addr:    server.Socket(),
+	})
+
+	for i := 0; i < b.N; i++ {
+		client.ZAddNX(context.Background(), "/yyy/bin", &redis.Z{Score: 0, Member: strconv.Itoa(i) + "namexxxxxxxxxxxxxxxxxxx"})
+	}
+}
+
+func xTestNameListAdd(t *testing.T) {
+
+	server, err := tempredis.Start(tempredis.Config{})
+	if err != nil {
+		panic(err)
+	}
+	defer server.Term()
+
+	client := redis.NewClient(&redis.Options{
+		Addr:     "localhost:6379",
+		Password: "",
+		DB:       0,
+	})
+
+	client.FlushAll(context.Background())
+
+	N := 364800
+
+	ts0 := time.Now()
+	store := newSkipListElementStore("/y", client)
+	var data []byte
+	nameList := LoadItemList(data, "/y", client, store, 100000)
+	for i := 0; i < N; i++ {
+		nameList.WriteName(fmt.Sprintf("%8d", i))
+	}
+
+	ts1 := time.Now()
+
+	for i := 0; i < N; i++ {
+		client.ZAddNX(context.Background(), "/x", &redis.Z{Score: 0, Member: fmt.Sprintf("name %8d", i)})
+	}
+	ts2 := time.Now()
+
+	fmt.Printf("%v %v", ts1.Sub(ts0), ts2.Sub(ts1))
+
+	/*
+		keys := client.Keys(context.Background(), "/*m").Val()
+		for _, k := range keys {
+			println("key", k)
+			for i, v := range client.ZRangeByLex(context.Background(), k, &redis.ZRangeBy{
+				Min:    "-",
+				Max:    "+",
+			}).Val() {
+				println(" ", i, v)
+			}
+		}
+	*/
+}
+
+func xBenchmarkNameList(b *testing.B) {
+
+	server, err := tempredis.Start(tempredis.Config{})
+	if err != nil {
+		panic(err)
+	}
+	defer server.Term()
+
+	client := redis.NewClient(&redis.Options{
+		Addr:     "localhost:6379",
+		Password: "",
+		DB:       0,
+	})
+
+	store := newSkipListElementStore("/yyy/bin", client)
+	var data []byte
+	for i := 0; i < b.N; i++ {
+		nameList := LoadItemList(data, "/yyy/bin", client, store, maxNameBatchSizeLimit)
+
+		nameList.WriteName(fmt.Sprintf("name %8d", i))
+
+		if nameList.HasChanges() {
+			data = nameList.ToBytes()
+		}
+	}
+}
+
+func xBenchmarkRedis(b *testing.B) {
+
+	client := redis.NewClient(&redis.Options{
+		Addr:     "localhost:6379",
+		Password: "",
+		DB:       0,
+	})
+
+	for i := 0; i < b.N; i++ {
+		client.ZAddNX(context.Background(), "/xxx/bin", &redis.Z{Score: 0, Member: fmt.Sprintf("name %8d", i)})
+	}
+}

weed/filer/redis3/redis_cluster_store.go

@@ -0,0 +1,45 @@
+package redis3
+
+import (
+	"github.com/chrislusf/seaweedfs/weed/filer"
+	"github.com/chrislusf/seaweedfs/weed/util"
+	"github.com/go-redis/redis/v8"
+	"github.com/go-redsync/redsync/v4"
+	"github.com/go-redsync/redsync/v4/redis/goredis/v8"
+)
+
+func init() {
+	filer.Stores = append(filer.Stores, &RedisCluster3Store{})
+}
+
+type RedisCluster3Store struct {
+	UniversalRedis3Store
+}
+
+func (store *RedisCluster3Store) GetName() string {
+	return "redis_cluster3"
+}
+
+func (store *RedisCluster3Store) Initialize(configuration util.Configuration, prefix string) (err error) {
+
+	configuration.SetDefault(prefix+"useReadOnly", false)
+	configuration.SetDefault(prefix+"routeByLatency", false)
+
+	return store.initialize(
+		configuration.GetStringSlice(prefix+"addresses"),
+		configuration.GetString(prefix+"password"),
+		configuration.GetBool(prefix+"useReadOnly"),
+		configuration.GetBool(prefix+"routeByLatency"),
+	)
+}
+
+func (store *RedisCluster3Store) initialize(addresses []string, password string, readOnly, routeByLatency bool) (err error) {
+	store.Client = redis.NewClusterClient(&redis.ClusterOptions{
+		Addrs:          addresses,
+		Password:       password,
+		ReadOnly:       readOnly,
+		RouteByLatency: routeByLatency,
+	})
+	store.redsync = redsync.New(goredis.NewPool(store.Client))
+	return
+}

weed/filer/redis3/redis_store.go

@@ -0,0 +1,39 @@
+package redis3
+
+import (
+	"github.com/chrislusf/seaweedfs/weed/filer"
+	"github.com/chrislusf/seaweedfs/weed/util"
+	"github.com/go-redis/redis/v8"
+	"github.com/go-redsync/redsync/v4"
+	"github.com/go-redsync/redsync/v4/redis/goredis/v8"
+)
+
+func init() {
+	filer.Stores = append(filer.Stores, &Redis3Store{})
+}
+
+type Redis3Store struct {
+	UniversalRedis3Store
+}
+
+func (store *Redis3Store) GetName() string {
+	return "redis3"
+}
+
+func (store *Redis3Store) Initialize(configuration util.Configuration, prefix string) (err error) {
+	return store.initialize(
+		configuration.GetString(prefix+"address"),
+		configuration.GetString(prefix+"password"),
+		configuration.GetInt(prefix+"database"),
+	)
+}
+
+func (store *Redis3Store) initialize(hostPort string, password string, database int) (err error) {
+	store.Client = redis.NewClient(&redis.Options{
+		Addr:     hostPort,
+		Password: password,
+		DB:       database,
+	})
+	store.redsync = redsync.New(goredis.NewPool(store.Client))
+	return
+}

weed/filer/redis3/skiplist_element_store.go

@@ -0,0 +1,62 @@
+package redis3
+
+import (
+	"context"
+	"fmt"
+	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/util/skiplist"
+	"github.com/go-redis/redis/v8"
+	"github.com/golang/protobuf/proto"
+)
+
+type SkipListElementStore struct {
+	Prefix string
+	client redis.UniversalClient
+}
+
+var _ = skiplist.ListStore(&SkipListElementStore{})
+
+func newSkipListElementStore(prefix string, client redis.UniversalClient) *SkipListElementStore {
+	return &SkipListElementStore{
+		Prefix: prefix,
+		client: client,
+	}
+}
+
+func (m *SkipListElementStore) SaveElement(id int64, element *skiplist.SkipListElement) error {
+	key := fmt.Sprintf("%s%d", m.Prefix, id)
+	data, err := proto.Marshal(element)
+	if err != nil {
+		glog.Errorf("marshal %s: %v", key, err)
+	}
+	return m.client.Set(context.Background(), key, data, 0).Err()
+}
+
+func (m *SkipListElementStore) DeleteElement(id int64) error {
+	key := fmt.Sprintf("%s%d", m.Prefix, id)
+	return m.client.Del(context.Background(), key).Err()
+}
+
+func (m *SkipListElementStore) LoadElement(id int64) (*skiplist.SkipListElement, error) {
+	key := fmt.Sprintf("%s%d", m.Prefix, id)
+	data, err := m.client.Get(context.Background(), key).Result()
+	if err != nil {
+		if err == redis.Nil {
+			return nil, nil
+		}
+		return nil, err
+	}
+	t := &skiplist.SkipListElement{}
+	err = proto.Unmarshal([]byte(data), t)
+	if err == nil {
+		for i := 0; i < len(t.Next); i++ {
+			if t.Next[i].IsNil() {
+				t.Next[i] = nil
+			}
+		}
+		if t.Prev.IsNil() {
+			t.Prev = nil
+		}
+	}
+	return t, err
+}

weed/filer/redis3/universal_redis_store.go

@@ -0,0 +1,179 @@
+package redis3
+
+import (
+	"context"
+	"fmt"
+	"github.com/go-redsync/redsync/v4"
+	"time"
+
+	"github.com/go-redis/redis/v8"
+
+	"github.com/chrislusf/seaweedfs/weed/filer"
+	"github.com/chrislusf/seaweedfs/weed/glog"
+	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
+	"github.com/chrislusf/seaweedfs/weed/util"
+)
+
+const (
+	DIR_LIST_MARKER = "\x00"
+)
+
+type UniversalRedis3Store struct {
+	Client  redis.UniversalClient
+	redsync *redsync.Redsync
+}
+
+func (store *UniversalRedis3Store) BeginTransaction(ctx context.Context) (context.Context, error) {
+	return ctx, nil
+}
+func (store *UniversalRedis3Store) CommitTransaction(ctx context.Context) error {
+	return nil
+}
+func (store *UniversalRedis3Store) RollbackTransaction(ctx context.Context) error {
+	return nil
+}
+
+func (store *UniversalRedis3Store) InsertEntry(ctx context.Context, entry *filer.Entry) (err error) {
+
+	value, err := entry.EncodeAttributesAndChunks()
+	if err != nil {
+		return fmt.Errorf("encoding %s %+v: %v", entry.FullPath, entry.Attr, err)
+	}
+
+	if len(entry.Chunks) > 50 {
+		value = util.MaybeGzipData(value)
+	}
+
+	if err = store.Client.Set(ctx, string(entry.FullPath), value, time.Duration(entry.TtlSec)*time.Second).Err(); err != nil {
+		return fmt.Errorf("persisting %s : %v", entry.FullPath, err)
+	}
+
+	dir, name := entry.FullPath.DirAndName()
+
+	if name != "" {
+		if err = insertChild(ctx, store, genDirectoryListKey(dir), name); err != nil {
+			return fmt.Errorf("persisting %s in parent dir: %v", entry.FullPath, err)
+		}
+	}
+
+	return nil
+}
+
+func (store *UniversalRedis3Store) UpdateEntry(ctx context.Context, entry *filer.Entry) (err error) {
+
+	return store.InsertEntry(ctx, entry)
+}
+
+func (store *UniversalRedis3Store) FindEntry(ctx context.Context, fullpath util.FullPath) (entry *filer.Entry, err error) {
+
+	data, err := store.Client.Get(ctx, string(fullpath)).Result()
+	if err == redis.Nil {
+		return nil, filer_pb.ErrNotFound
+	}
+
+	if err != nil {
+		return nil, fmt.Errorf("get %s : %v", fullpath, err)
+	}
+
+	entry = &filer.Entry{
+		FullPath: fullpath,
+	}
+	err = entry.DecodeAttributesAndChunks(util.MaybeDecompressData([]byte(data)))
+	if err != nil {
+		return entry, fmt.Errorf("decode %s : %v", entry.FullPath, err)
+	}
+
+	return entry, nil
+}
+
+func (store *UniversalRedis3Store) DeleteEntry(ctx context.Context, fullpath util.FullPath) (err error) {
+
+	_, err = store.Client.Del(ctx, genDirectoryListKey(string(fullpath))).Result()
+	if err != nil {
+		return fmt.Errorf("delete dir list %s : %v", fullpath, err)
+	}
+
+	_, err = store.Client.Del(ctx, string(fullpath)).Result()
+	if err != nil {
+		return fmt.Errorf("delete %s : %v", fullpath, err)
+	}
+
+	dir, name := fullpath.DirAndName()
+
+	if name != "" {
+		if err = removeChild(ctx, store, genDirectoryListKey(dir), name); err != nil {
+			return fmt.Errorf("DeleteEntry %s in parent dir: %v", fullpath, err)
+		}
+	}
+
+	return nil
+}
+
+func (store *UniversalRedis3Store) DeleteFolderChildren(ctx context.Context, fullpath util.FullPath) (err error) {
+
+	return removeChildren(ctx, store, genDirectoryListKey(string(fullpath)), func(name string) error {
+		path := util.NewFullPath(string(fullpath), name)
+		_, err = store.Client.Del(ctx, string(path)).Result()
+		if err != nil {
+			return fmt.Errorf("DeleteFolderChildren %s in parent dir: %v", fullpath, err)
+		}
+		// not efficient, but need to remove if it is a directory
+		store.Client.Del(ctx, genDirectoryListKey(string(path)))
+		return nil
+	})
+
+}
+
+func (store *UniversalRedis3Store) ListDirectoryPrefixedEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, prefix string, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) {
+	return lastFileName, filer.ErrUnsupportedListDirectoryPrefixed
+}
+
+func (store *UniversalRedis3Store) ListDirectoryEntries(ctx context.Context, dirPath util.FullPath, startFileName string, includeStartFile bool, limit int64, eachEntryFunc filer.ListEachEntryFunc) (lastFileName string, err error) {
+
+	dirListKey := genDirectoryListKey(string(dirPath))
+	counter := int64(0)
+
+	err = listChildren(ctx, store, dirListKey, startFileName, func(fileName string) bool {
+		if startFileName != "" {
+			if !includeStartFile && startFileName == fileName {
+				return true
+			}
+		}
+
+		path := util.NewFullPath(string(dirPath), fileName)
+		entry, err := store.FindEntry(ctx, path)
+		lastFileName = fileName
+		if err != nil {
+			glog.V(0).Infof("list %s : %v", path, err)
+			if err == filer_pb.ErrNotFound {
+				return true
+			}
+		} else {
+			if entry.TtlSec > 0 {
+				if entry.Attr.Crtime.Add(time.Duration(entry.TtlSec) * time.Second).Before(time.Now()) {
+					store.Client.Del(ctx, string(path)).Result()
+					store.Client.ZRem(ctx, dirListKey, fileName).Result()
+					return true
+				}
+			}
+			counter++
+			if !eachEntryFunc(entry) {
+				return false
+			}
+			if counter >= limit {
+				return false
+			}
+		}
+		return true
+	})
+
+	return lastFileName, err
+}
+
+func genDirectoryListKey(dir string) (dirList string) {
+	return dir + DIR_LIST_MARKER
+}
+
+func (store *UniversalRedis3Store) Shutdown() {
+	store.Client.Close()
+}

weed/filer/redis3/universal_redis_store_kv.go

@@ -0,0 +1,42 @@
+package redis3
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/chrislusf/seaweedfs/weed/filer"
+	"github.com/go-redis/redis/v8"
+)
+
+func (store *UniversalRedis3Store) KvPut(ctx context.Context, key []byte, value []byte) (err error) {
+
+	_, err = store.Client.Set(ctx, string(key), value, 0).Result()
+
+	if err != nil {
+		return fmt.Errorf("kv put: %v", err)
+	}
+
+	return nil
+}
+
+func (store *UniversalRedis3Store) KvGet(ctx context.Context, key []byte) (value []byte, err error) {
+
+	data, err := store.Client.Get(ctx, string(key)).Result()
+
+	if err == redis.Nil {
+		return nil, filer.ErrKvNotFound
+	}
+
+	return []byte(data), err
+}
+
+func (store *UniversalRedis3Store) KvDelete(ctx context.Context, key []byte) (err error) {
+
+	_, err = store.Client.Del(ctx, string(key)).Result()
+
+	if err != nil {
+		return fmt.Errorf("kv delete: %v", err)
+	}
+
+	return nil
+}

weed/filer/remote_mapping.go

@@ -9,7 +9,7 @@ import (
 	"google.golang.org/grpc"
 )
 
-func ReadMountMappings(grpcDialOption grpc.DialOption, filerAddress string) (mappings *remote_pb.RemoteStorageMapping, readErr error) {
+func ReadMountMappings(grpcDialOption grpc.DialOption, filerAddress pb.ServerAddress) (mappings *remote_pb.RemoteStorageMapping, readErr error) {
 	var oldContent []byte
 	if readErr = pb.WithFilerClient(filerAddress, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
 		oldContent, readErr = ReadInsideFiler(client, DirectoryEtcRemote, REMOTE_STORAGE_MOUNT_FILE)

weed/filer/remote_storage.go

@@ -131,7 +131,7 @@ func UnmarshalRemoteStorageMappings(oldContent []byte) (mappings *remote_pb.Remo
 	return
 }
 
-func ReadRemoteStorageConf(grpcDialOption grpc.DialOption, filerAddress string, storageName string) (conf *remote_pb.RemoteConf, readErr error) {
+func ReadRemoteStorageConf(grpcDialOption grpc.DialOption, filerAddress pb.ServerAddress, storageName string) (conf *remote_pb.RemoteConf, readErr error) {
 	var oldContent []byte
 	if readErr = pb.WithFilerClient(filerAddress, grpcDialOption, func(client filer_pb.SeaweedFilerClient) error {
 		oldContent, readErr = ReadInsideFiler(client, DirectoryEtcRemote, storageName+REMOTE_STORAGE_CONF_SUFFIX)
@@ -150,7 +150,7 @@ func ReadRemoteStorageConf(grpcDialOption grpc.DialOption, filerAddress string,
 	return
 }
 
-func DetectMountInfo(grpcDialOption grpc.DialOption, filerAddress string, dir string) (*remote_pb.RemoteStorageMapping, string, *remote_pb.RemoteStorageLocation, *remote_pb.RemoteConf, error) {
+func DetectMountInfo(grpcDialOption grpc.DialOption, filerAddress pb.ServerAddress, dir string) (*remote_pb.RemoteStorageMapping, string, *remote_pb.RemoteStorageLocation, *remote_pb.RemoteConf, error) {
 
 	mappings, listErr := ReadMountMappings(grpcDialOption, filerAddress)
 	if listErr != nil {

weed/filer/rocksdb/rocksdb_store.go

@@ -10,7 +10,7 @@ import (
 	"io"
 	"os"
 
-	"github.com/tecbot/gorocksdb"
+	gorocksdb "github.com/linxGnu/grocksdb"
 
 	"github.com/chrislusf/seaweedfs/weed/filer"
 	"github.com/chrislusf/seaweedfs/weed/glog"