Browse Source

diff

pull/2764/head
a 3 years ago
parent
commit
bc603e534f
  1. 10
      .github/dependabot.yml
  2. 14
      .github/workflows/binaries_dev.yml
  3. 6
      .github/workflows/binaries_release0.yml
  4. 6
      .github/workflows/binaries_release1.yml
  5. 6
      .github/workflows/binaries_release2.yml
  6. 6
      .github/workflows/binaries_release3.yml
  7. 17
      .github/workflows/container_dev.yml
  8. 17
      .github/workflows/container_latest.yml
  9. 15
      .github/workflows/container_release1.yml
  10. 15
      .github/workflows/container_release2.yml
  11. 15
      .github/workflows/container_release3.yml
  12. 7
      .github/workflows/go.yml
  13. 2
      README.md
  14. 10
      docker/compose/local-cluster-compose.yml
  15. 128
      go.mod
  16. 727
      go.sum
  17. 4
      k8s/helm_charts2/Chart.yaml
  18. 2
      weed/command/benchmark.go
  19. 7
      weed/command/filer.go
  20. 2
      weed/command/iam.go
  21. 11
      weed/command/master.go
  22. 15
      weed/command/master_follower.go
  23. 2
      weed/command/server.go
  24. 8
      weed/filer/filechunks.go
  25. 2
      weed/filer/filer.go
  26. 3
      weed/filer/meta_aggregator.go
  27. 1
      weed/filer/read_write.go
  28. 2
      weed/iamapi/iamapi_server.go
  29. 7
      weed/mount/weedfs_file_mkrm.go
  30. 7
      weed/mount/weedfs_file_sync.go
  31. 2
      weed/pb/grpc_client_server.go
  32. 4
      weed/pb/remote.proto
  33. 105
      weed/pb/remote_pb/remote.pb.go
  34. 15
      weed/pb/server_address.go
  35. 50
      weed/remote_storage/s3/contabo.go
  36. 30
      weed/s3api/filer_multipart.go
  37. 83
      weed/s3api/filer_multipart_test.go
  38. 9
      weed/s3api/filer_util.go
  39. 36
      weed/s3api/s3api_object_multipart_handlers.go
  40. 2
      weed/server/filer_grpc_server.go
  41. 2
      weed/server/filer_server.go
  42. 4
      weed/server/filer_server_handlers_tagging.go
  43. 2
      weed/server/filer_server_handlers_write_autochunk.go
  44. 2
      weed/server/master_server.go
  45. 40
      weed/server/raft_server.go
  46. 2
      weed/server/volume_server.go
  47. 25
      weed/server/volume_server_handlers.go
  48. 2
      weed/shell/commands.go
  49. 2
      weed/storage/erasure_coding/ec_encoder.go
  50. 13
      weed/topology/volume_layout.go
  51. 25
      weed/util/config.go
  52. 2
      weed/util/constants.go
  53. 2
      weed/util/net_timeout.go
  54. 4
      weed/wdclient/masterclient.go
  55. 3
      weed/weed.go

10
.github/dependabot.yml

@ -0,0 +1,10 @@
version: 2
updates:
- package-ecosystem: "github-actions"
directory: "/"
schedule:
interval: "daily"
- package-ecosystem: gomod
directory: "/"
schedule:
interval: daily

14
.github/workflows/binaries_dev.yml

@ -12,7 +12,7 @@ jobs:
steps: steps:
- name: Delete old release assets - name: Delete old release assets
uses: mknejp/delete-release-assets@v1
uses: mknejp/delete-release-assets@a8aaab13272b1eaac16cc46dddd3f725b97ee05a # v1
with: with:
token: ${{ github.token }} token: ${{ github.token }}
tag: dev tag: dev
@ -31,13 +31,13 @@ jobs:
steps: steps:
- name: Check out code into the Go module directory - name: Check out code into the Go module directory
uses: actions/checkout@v2
uses: actions/checkout@629c2de402a417ea7690ca6ce3f33229e27606a5 # v2
- name: Set BUILD_TIME env - name: Set BUILD_TIME env
run: echo BUILD_TIME=$(date -u +%Y%m%d-%H%M) >> ${GITHUB_ENV} run: echo BUILD_TIME=$(date -u +%Y%m%d-%H%M) >> ${GITHUB_ENV}
- name: Go Release Binaries Large Disk - name: Go Release Binaries Large Disk
uses: wangyoucao577/go-release-action@v1.22
uses: wangyoucao577/go-release-action@16624612d4e2b73de613857a362d294700207fff # v1.22
with: with:
github_token: ${{ secrets.GITHUB_TOKEN }} github_token: ${{ secrets.GITHUB_TOKEN }}
goos: ${{ matrix.goos }} goos: ${{ matrix.goos }}
@ -53,7 +53,7 @@ jobs:
asset_name: "weed-large-disk-${{ env.BUILD_TIME }}-${{ matrix.goos }}-${{ matrix.goarch }}" asset_name: "weed-large-disk-${{ env.BUILD_TIME }}-${{ matrix.goos }}-${{ matrix.goarch }}"
- name: Go Release Binaries Normal Volume Size - name: Go Release Binaries Normal Volume Size
uses: wangyoucao577/go-release-action@v1.22
uses: wangyoucao577/go-release-action@16624612d4e2b73de613857a362d294700207fff # v1.22
with: with:
github_token: ${{ secrets.GITHUB_TOKEN }} github_token: ${{ secrets.GITHUB_TOKEN }}
goos: ${{ matrix.goos }} goos: ${{ matrix.goos }}
@ -78,13 +78,13 @@ jobs:
steps: steps:
- name: Check out code into the Go module directory - name: Check out code into the Go module directory
uses: actions/checkout@v2
uses: actions/checkout@629c2de402a417ea7690ca6ce3f33229e27606a5 # v2
- name: Set BUILD_TIME env - name: Set BUILD_TIME env
run: echo BUILD_TIME=$(date -u +%Y%m%d-%H%M) >> ${GITHUB_ENV} run: echo BUILD_TIME=$(date -u +%Y%m%d-%H%M) >> ${GITHUB_ENV}
- name: Go Release Binaries Large Disk - name: Go Release Binaries Large Disk
uses: wangyoucao577/go-release-action@v1.22
uses: wangyoucao577/go-release-action@16624612d4e2b73de613857a362d294700207fff # v1.22
with: with:
github_token: ${{ secrets.GITHUB_TOKEN }} github_token: ${{ secrets.GITHUB_TOKEN }}
goos: ${{ matrix.goos }} goos: ${{ matrix.goos }}
@ -100,7 +100,7 @@ jobs:
asset_name: "weed-large-disk-${{ env.BUILD_TIME }}-${{ matrix.goos }}-${{ matrix.goarch }}" asset_name: "weed-large-disk-${{ env.BUILD_TIME }}-${{ matrix.goos }}-${{ matrix.goarch }}"
- name: Go Release Binaries Normal Volume Size - name: Go Release Binaries Normal Volume Size
uses: wangyoucao577/go-release-action@v1.22
uses: wangyoucao577/go-release-action@16624612d4e2b73de613857a362d294700207fff # v1.22
with: with:
github_token: ${{ secrets.GITHUB_TOKEN }} github_token: ${{ secrets.GITHUB_TOKEN }}
goos: ${{ matrix.goos }} goos: ${{ matrix.goos }}

6
.github/workflows/binaries_release0.yml

@ -23,9 +23,9 @@ jobs:
# Steps represent a sequence of tasks that will be executed as part of the job # Steps represent a sequence of tasks that will be executed as part of the job
steps: steps:
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
- uses: actions/checkout@v2
- uses: actions/checkout@629c2de402a417ea7690ca6ce3f33229e27606a5 # v2
- name: Go Release Binaries Normal Volume Size - name: Go Release Binaries Normal Volume Size
uses: wangyoucao577/go-release-action@v1.22
uses: wangyoucao577/go-release-action@16624612d4e2b73de613857a362d294700207fff # v1.22
with: with:
github_token: ${{ secrets.GITHUB_TOKEN }} github_token: ${{ secrets.GITHUB_TOKEN }}
goos: ${{ matrix.goos }} goos: ${{ matrix.goos }}
@ -39,7 +39,7 @@ jobs:
binary_name: weed binary_name: weed
asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}" asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
- name: Go Release Large Disk Binaries - name: Go Release Large Disk Binaries
uses: wangyoucao577/go-release-action@v1.22
uses: wangyoucao577/go-release-action@16624612d4e2b73de613857a362d294700207fff # v1.22
with: with:
github_token: ${{ secrets.GITHUB_TOKEN }} github_token: ${{ secrets.GITHUB_TOKEN }}
goos: ${{ matrix.goos }} goos: ${{ matrix.goos }}

6
.github/workflows/binaries_release1.yml

@ -23,9 +23,9 @@ jobs:
# Steps represent a sequence of tasks that will be executed as part of the job # Steps represent a sequence of tasks that will be executed as part of the job
steps: steps:
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
- uses: actions/checkout@v2
- uses: actions/checkout@629c2de402a417ea7690ca6ce3f33229e27606a5 # v2
- name: Go Release Binaries Normal Volume Size - name: Go Release Binaries Normal Volume Size
uses: wangyoucao577/go-release-action@v1.22
uses: wangyoucao577/go-release-action@16624612d4e2b73de613857a362d294700207fff # v1.22
with: with:
github_token: ${{ secrets.GITHUB_TOKEN }} github_token: ${{ secrets.GITHUB_TOKEN }}
goos: ${{ matrix.goos }} goos: ${{ matrix.goos }}
@ -39,7 +39,7 @@ jobs:
binary_name: weed binary_name: weed
asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}" asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
- name: Go Release Large Disk Binaries - name: Go Release Large Disk Binaries
uses: wangyoucao577/go-release-action@v1.22
uses: wangyoucao577/go-release-action@16624612d4e2b73de613857a362d294700207fff # v1.22
with: with:
github_token: ${{ secrets.GITHUB_TOKEN }} github_token: ${{ secrets.GITHUB_TOKEN }}
goos: ${{ matrix.goos }} goos: ${{ matrix.goos }}

6
.github/workflows/binaries_release2.yml

@ -23,9 +23,9 @@ jobs:
# Steps represent a sequence of tasks that will be executed as part of the job # Steps represent a sequence of tasks that will be executed as part of the job
steps: steps:
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
- uses: actions/checkout@v2
- uses: actions/checkout@629c2de402a417ea7690ca6ce3f33229e27606a5 # v2
- name: Go Release Binaries Normal Volume Size - name: Go Release Binaries Normal Volume Size
uses: wangyoucao577/go-release-action@v1.22
uses: wangyoucao577/go-release-action@16624612d4e2b73de613857a362d294700207fff # v1.22
with: with:
github_token: ${{ secrets.GITHUB_TOKEN }} github_token: ${{ secrets.GITHUB_TOKEN }}
goos: ${{ matrix.goos }} goos: ${{ matrix.goos }}
@ -39,7 +39,7 @@ jobs:
binary_name: weed binary_name: weed
asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}" asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
- name: Go Release Large Disk Binaries - name: Go Release Large Disk Binaries
uses: wangyoucao577/go-release-action@v1.22
uses: wangyoucao577/go-release-action@16624612d4e2b73de613857a362d294700207fff # v1.22
with: with:
github_token: ${{ secrets.GITHUB_TOKEN }} github_token: ${{ secrets.GITHUB_TOKEN }}
goos: ${{ matrix.goos }} goos: ${{ matrix.goos }}

6
.github/workflows/binaries_release3.yml

@ -23,9 +23,9 @@ jobs:
# Steps represent a sequence of tasks that will be executed as part of the job # Steps represent a sequence of tasks that will be executed as part of the job
steps: steps:
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
- uses: actions/checkout@v2
- uses: actions/checkout@629c2de402a417ea7690ca6ce3f33229e27606a5 # v2
- name: Go Release Binaries Normal Volume Size - name: Go Release Binaries Normal Volume Size
uses: wangyoucao577/go-release-action@v1.22
uses: wangyoucao577/go-release-action@16624612d4e2b73de613857a362d294700207fff # v1.22
with: with:
github_token: ${{ secrets.GITHUB_TOKEN }} github_token: ${{ secrets.GITHUB_TOKEN }}
goos: ${{ matrix.goos }} goos: ${{ matrix.goos }}
@ -39,7 +39,7 @@ jobs:
binary_name: weed binary_name: weed
asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}" asset_name: "${{ matrix.goos }}_${{ matrix.goarch }}"
- name: Go Release Large Disk Binaries - name: Go Release Large Disk Binaries
uses: wangyoucao577/go-release-action@v1.22
uses: wangyoucao577/go-release-action@16624612d4e2b73de613857a362d294700207fff # v1.22
with: with:
github_token: ${{ secrets.GITHUB_TOKEN }} github_token: ${{ secrets.GITHUB_TOKEN }}
goos: ${{ matrix.goos }} goos: ${{ matrix.goos }}

17
.github/workflows/container_dev.yml

@ -5,6 +5,9 @@ on:
branches: [ master ] branches: [ master ]
workflow_dispatch: [] workflow_dispatch: []
permissions:
contents: read
jobs: jobs:
build-dev-containers: build-dev-containers:
@ -13,11 +16,11 @@ jobs:
steps: steps:
- -
name: Checkout name: Checkout
uses: actions/checkout@v2
uses: actions/checkout@629c2de402a417ea7690ca6ce3f33229e27606a5 # v2
- -
name: Docker meta name: Docker meta
id: docker_meta id: docker_meta
uses: docker/metadata-action@v3
uses: docker/metadata-action@e5622373a38e60fb6d795a4421e56882f2d7a681 # v3
with: with:
images: | images: |
chrislusf/seaweedfs chrislusf/seaweedfs
@ -30,30 +33,30 @@ jobs:
org.opencontainers.image.vendor=Chris Lu org.opencontainers.image.vendor=Chris Lu
- -
name: Set up QEMU name: Set up QEMU
uses: docker/setup-qemu-action@v1
uses: docker/setup-qemu-action@27d0a4f181a40b142cce983c5393082c365d1480 # v1
- -
name: Set up Docker Buildx name: Set up Docker Buildx
uses: docker/setup-buildx-action@v1
uses: docker/setup-buildx-action@94ab11c41e45d028884a99163086648e898eed25 # v1
with: with:
buildkitd-flags: "--debug" buildkitd-flags: "--debug"
- -
name: Login to Docker Hub name: Login to Docker Hub
if: github.event_name != 'pull_request' if: github.event_name != 'pull_request'
uses: docker/login-action@v1
uses: docker/login-action@dd4fa0671be5250ee6f50aedf4cb05514abda2c7 # v1
with: with:
username: ${{ secrets.DOCKER_USERNAME }} username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }} password: ${{ secrets.DOCKER_PASSWORD }}
- -
name: Login to GHCR name: Login to GHCR
if: github.event_name != 'pull_request' if: github.event_name != 'pull_request'
uses: docker/login-action@v1
uses: docker/login-action@dd4fa0671be5250ee6f50aedf4cb05514abda2c7 # v1
with: with:
registry: ghcr.io registry: ghcr.io
username: ${{ secrets.GHCR_USERNAME }} username: ${{ secrets.GHCR_USERNAME }}
password: ${{ secrets.GHCR_TOKEN }} password: ${{ secrets.GHCR_TOKEN }}
- -
name: Build name: Build
uses: docker/build-push-action@v2
uses: docker/build-push-action@ac9327eae2b366085ac7f6a2d02df8aa8ead720a # v2
with: with:
context: ./docker context: ./docker
push: ${{ github.event_name != 'pull_request' }} push: ${{ github.event_name != 'pull_request' }}

17
.github/workflows/container_latest.yml

@ -6,6 +6,9 @@ on:
- '*' - '*'
workflow_dispatch: [] workflow_dispatch: []
permissions:
contents: read
jobs: jobs:
build-latest-container: build-latest-container:
@ -14,11 +17,11 @@ jobs:
steps: steps:
- -
name: Checkout name: Checkout
uses: actions/checkout@v2
uses: actions/checkout@629c2de402a417ea7690ca6ce3f33229e27606a5 # v2
- -
name: Docker meta name: Docker meta
id: docker_meta id: docker_meta
uses: docker/metadata-action@v3
uses: docker/metadata-action@e5622373a38e60fb6d795a4421e56882f2d7a681 # v3
with: with:
images: | images: |
chrislusf/seaweedfs chrislusf/seaweedfs
@ -31,30 +34,30 @@ jobs:
org.opencontainers.image.vendor=Chris Lu org.opencontainers.image.vendor=Chris Lu
- -
name: Set up QEMU name: Set up QEMU
uses: docker/setup-qemu-action@v1
uses: docker/setup-qemu-action@27d0a4f181a40b142cce983c5393082c365d1480 # v1
- -
name: Set up Docker Buildx name: Set up Docker Buildx
uses: docker/setup-buildx-action@v1
uses: docker/setup-buildx-action@94ab11c41e45d028884a99163086648e898eed25 # v1
with: with:
buildkitd-flags: "--debug" buildkitd-flags: "--debug"
- -
name: Login to Docker Hub name: Login to Docker Hub
if: github.event_name != 'pull_request' if: github.event_name != 'pull_request'
uses: docker/login-action@v1
uses: docker/login-action@dd4fa0671be5250ee6f50aedf4cb05514abda2c7 # v1
with: with:
username: ${{ secrets.DOCKER_USERNAME }} username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }} password: ${{ secrets.DOCKER_PASSWORD }}
- -
name: Login to GHCR name: Login to GHCR
if: github.event_name != 'pull_request' if: github.event_name != 'pull_request'
uses: docker/login-action@v1
uses: docker/login-action@dd4fa0671be5250ee6f50aedf4cb05514abda2c7 # v1
with: with:
registry: ghcr.io registry: ghcr.io
username: ${{ secrets.GHCR_USERNAME }} username: ${{ secrets.GHCR_USERNAME }}
password: ${{ secrets.GHCR_TOKEN }} password: ${{ secrets.GHCR_TOKEN }}
- -
name: Build name: Build
uses: docker/build-push-action@v2
uses: docker/build-push-action@ac9327eae2b366085ac7f6a2d02df8aa8ead720a # v2
with: with:
context: ./docker context: ./docker
push: ${{ github.event_name != 'pull_request' }} push: ${{ github.event_name != 'pull_request' }}

15
.github/workflows/container_release1.yml

@ -6,6 +6,9 @@ on:
- '*' - '*'
workflow_dispatch: [] workflow_dispatch: []
permissions:
contents: read
jobs: jobs:
build-default-release-container: build-default-release-container:
runs-on: [ubuntu-latest] runs-on: [ubuntu-latest]
@ -13,11 +16,11 @@ jobs:
steps: steps:
- -
name: Checkout name: Checkout
uses: actions/checkout@v2
uses: actions/checkout@629c2de402a417ea7690ca6ce3f33229e27606a5 # v2
- -
name: Docker meta name: Docker meta
id: docker_meta id: docker_meta
uses: docker/metadata-action@v3
uses: docker/metadata-action@e5622373a38e60fb6d795a4421e56882f2d7a681 # v3
with: with:
images: | images: |
chrislusf/seaweedfs chrislusf/seaweedfs
@ -31,20 +34,20 @@ jobs:
org.opencontainers.image.vendor=Chris Lu org.opencontainers.image.vendor=Chris Lu
- -
name: Set up QEMU name: Set up QEMU
uses: docker/setup-qemu-action@v1
uses: docker/setup-qemu-action@27d0a4f181a40b142cce983c5393082c365d1480 # v1
- -
name: Set up Docker Buildx name: Set up Docker Buildx
uses: docker/setup-buildx-action@v1
uses: docker/setup-buildx-action@94ab11c41e45d028884a99163086648e898eed25 # v1
- -
name: Login to Docker Hub name: Login to Docker Hub
if: github.event_name != 'pull_request' if: github.event_name != 'pull_request'
uses: docker/login-action@v1
uses: docker/login-action@dd4fa0671be5250ee6f50aedf4cb05514abda2c7 # v1
with: with:
username: ${{ secrets.DOCKER_USERNAME }} username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }} password: ${{ secrets.DOCKER_PASSWORD }}
- -
name: Build name: Build
uses: docker/build-push-action@v2
uses: docker/build-push-action@ac9327eae2b366085ac7f6a2d02df8aa8ead720a # v2
with: with:
context: ./docker context: ./docker
push: ${{ github.event_name != 'pull_request' }} push: ${{ github.event_name != 'pull_request' }}

15
.github/workflows/container_release2.yml

@ -6,6 +6,9 @@ on:
- '*' - '*'
workflow_dispatch: [] workflow_dispatch: []
permissions:
contents: read
jobs: jobs:
build-large-release-container: build-large-release-container:
@ -14,11 +17,11 @@ jobs:
steps: steps:
- -
name: Checkout name: Checkout
uses: actions/checkout@v2
uses: actions/checkout@629c2de402a417ea7690ca6ce3f33229e27606a5 # v2
- -
name: Docker meta name: Docker meta
id: docker_meta id: docker_meta
uses: docker/metadata-action@v3
uses: docker/metadata-action@e5622373a38e60fb6d795a4421e56882f2d7a681 # v3
with: with:
images: | images: |
chrislusf/seaweedfs chrislusf/seaweedfs
@ -32,20 +35,20 @@ jobs:
org.opencontainers.image.vendor=Chris Lu org.opencontainers.image.vendor=Chris Lu
- -
name: Set up QEMU name: Set up QEMU
uses: docker/setup-qemu-action@v1
uses: docker/setup-qemu-action@27d0a4f181a40b142cce983c5393082c365d1480 # v1
- -
name: Set up Docker Buildx name: Set up Docker Buildx
uses: docker/setup-buildx-action@v1
uses: docker/setup-buildx-action@94ab11c41e45d028884a99163086648e898eed25 # v1
- -
name: Login to Docker Hub name: Login to Docker Hub
if: github.event_name != 'pull_request' if: github.event_name != 'pull_request'
uses: docker/login-action@v1
uses: docker/login-action@dd4fa0671be5250ee6f50aedf4cb05514abda2c7 # v1
with: with:
username: ${{ secrets.DOCKER_USERNAME }} username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }} password: ${{ secrets.DOCKER_PASSWORD }}
- -
name: Build name: Build
uses: docker/build-push-action@v2
uses: docker/build-push-action@ac9327eae2b366085ac7f6a2d02df8aa8ead720a # v2
with: with:
context: ./docker context: ./docker
push: ${{ github.event_name != 'pull_request' }} push: ${{ github.event_name != 'pull_request' }}

15
.github/workflows/container_release3.yml

@ -6,6 +6,9 @@ on:
- '*' - '*'
workflow_dispatch: [] workflow_dispatch: []
permissions:
contents: read
jobs: jobs:
build-large-release-container_rocksdb: build-large-release-container_rocksdb:
@ -14,11 +17,11 @@ jobs:
steps: steps:
- -
name: Checkout name: Checkout
uses: actions/checkout@v2
uses: actions/checkout@629c2de402a417ea7690ca6ce3f33229e27606a5 # v2
- -
name: Docker meta name: Docker meta
id: docker_meta id: docker_meta
uses: docker/metadata-action@v3
uses: docker/metadata-action@e5622373a38e60fb6d795a4421e56882f2d7a681 # v3
with: with:
images: | images: |
chrislusf/seaweedfs chrislusf/seaweedfs
@ -32,20 +35,20 @@ jobs:
org.opencontainers.image.vendor=Chris Lu org.opencontainers.image.vendor=Chris Lu
- -
name: Set up QEMU name: Set up QEMU
uses: docker/setup-qemu-action@v1
uses: docker/setup-qemu-action@27d0a4f181a40b142cce983c5393082c365d1480 # v1
- -
name: Set up Docker Buildx name: Set up Docker Buildx
uses: docker/setup-buildx-action@v1
uses: docker/setup-buildx-action@94ab11c41e45d028884a99163086648e898eed25 # v1
- -
name: Login to Docker Hub name: Login to Docker Hub
if: github.event_name != 'pull_request' if: github.event_name != 'pull_request'
uses: docker/login-action@v1
uses: docker/login-action@dd4fa0671be5250ee6f50aedf4cb05514abda2c7 # v1
with: with:
username: ${{ secrets.DOCKER_USERNAME }} username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }} password: ${{ secrets.DOCKER_PASSWORD }}
- -
name: Build name: Build
uses: docker/build-push-action@v2
uses: docker/build-push-action@ac9327eae2b366085ac7f6a2d02df8aa8ead720a # v2
with: with:
context: ./docker context: ./docker
push: ${{ github.event_name != 'pull_request' }} push: ${{ github.event_name != 'pull_request' }}

7
.github/workflows/go.yml

@ -10,6 +10,9 @@ concurrency:
group: ${{ github.head_ref }}/go group: ${{ github.head_ref }}/go
cancel-in-progress: true cancel-in-progress: true
permissions:
contents: read
jobs: jobs:
build: build:
@ -18,13 +21,13 @@ jobs:
steps: steps:
- name: Set up Go 1.x - name: Set up Go 1.x
uses: actions/setup-go@v2
uses: actions/setup-go@f6164bd8c8acb4a71fb2791a8b6c4024ff038dab # v2
with: with:
go-version: ^1.13 go-version: ^1.13
id: go id: go
- name: Check out code into the Go module directory - name: Check out code into the Go module directory
uses: actions/checkout@v2
uses: actions/checkout@629c2de402a417ea7690ca6ce3f33229e27606a5 # v2
- name: Get dependencies - name: Get dependencies
run: | run: |

2
README.md

@ -33,7 +33,7 @@ Your support will be really appreciated by me and other supporters!
### Gold Sponsors ### Gold Sponsors
![shuguang](https://raw.githubusercontent.com/chrislusf/seaweedfs/master/note/shuguang.png)
- [![nodion](https://www.nodion.com/img/logo.svg)](https://www.nodion.com)
--- ---

10
docker/compose/local-cluster-compose.yml

@ -36,7 +36,7 @@ services:
ports: ports:
- 8080:8080 - 8080:8080
- 18080:18080 - 18080:18080
command: 'volume -dataCenter dc1 -rack v1 -mserver="master0:9333,master1:9334,master2:9335" -port=8080 -ip=volume1 -publicUrl=localhost:8080 -preStopSeconds=1'
command: 'volume -dataCenter=dc1 -rack=v1 -mserver="master0:9333,master1:9334,master2:9335" -port=8080 -ip=volume1 -publicUrl=localhost:8080 -preStopSeconds=1'
depends_on: depends_on:
- master0 - master0
- master1 - master1
@ -46,7 +46,7 @@ services:
ports: ports:
- 8082:8082 - 8082:8082
- 18082:18082 - 18082:18082
command: 'volume -dataCenter dc2 -rack v2 -mserver="master0:9333,master1:9334,master2:9335" -port=8082 -ip=volume2 -publicUrl=localhost:8082 -preStopSeconds=1'
command: 'volume -dataCenter=dc2 -rack=v2 -mserver="master0:9333,master1:9334,master2:9335" -port=8082 -ip=volume2 -publicUrl=localhost:8082 -preStopSeconds=1'
depends_on: depends_on:
- master0 - master0
- master1 - master1
@ -56,7 +56,7 @@ services:
ports: ports:
- 8083:8083 - 8083:8083
- 18083:18083 - 18083:18083
command: 'volume -dataCenter dc3 -rack v3 -mserver="master0:9333,master1:9334,master2:9335" -port=8083 -ip=volume3 -publicUrl=localhost:8083 -preStopSeconds=1'
command: 'volume -dataCenter=dc3 -rack=v3 -mserver="master0:9333,master1:9334,master2:9335" -port=8083 -ip=volume3 -publicUrl=localhost:8083 -preStopSeconds=1'
depends_on: depends_on:
- master0 - master0
- master1 - master1
@ -67,7 +67,7 @@ services:
- 8888:8888 - 8888:8888
- 18888:18888 - 18888:18888
- 8111:8111 - 8111:8111
command: 'filer -defaultReplicaPlacement 100 -iam -master="master0:9333,master1:9334,master2:9335"'
command: 'filer -defaultReplicaPlacement=100 -iam -master="master0:9333,master1:9334,master2:9335"'
depends_on: depends_on:
- master0 - master0
- master1 - master1
@ -85,4 +85,4 @@ services:
- master2 - master2
- volume1 - volume1
- volume2 - volume2
- filer
- filer

128
go.mod

@ -3,21 +3,21 @@ module github.com/chrislusf/seaweedfs
go 1.18 go 1.18
require ( require (
cloud.google.com/go v0.94.1 // indirect
cloud.google.com/go/pubsub v1.3.1
cloud.google.com/go/storage v1.16.1
cloud.google.com/go v0.100.2 // indirect
cloud.google.com/go/pubsub v1.19.0
cloud.google.com/go/storage v1.21.0
github.com/Azure/azure-pipeline-go v0.2.3 github.com/Azure/azure-pipeline-go v0.2.3
github.com/Azure/azure-storage-blob-go v0.14.0 github.com/Azure/azure-storage-blob-go v0.14.0
github.com/OneOfOne/xxhash v1.2.2 github.com/OneOfOne/xxhash v1.2.2
github.com/Shopify/sarama v1.23.1 github.com/Shopify/sarama v1.23.1
github.com/aws/aws-sdk-go v1.35.3
github.com/aws/aws-sdk-go v1.43.25
github.com/beorn7/perks v1.0.1 // indirect github.com/beorn7/perks v1.0.1 // indirect
github.com/buraksezer/consistent v0.0.0-20191006190839-693edf70fd72 github.com/buraksezer/consistent v0.0.0-20191006190839-693edf70fd72
github.com/bwmarrin/snowflake v0.3.0 github.com/bwmarrin/snowflake v0.3.0
github.com/cespare/xxhash v1.1.0 github.com/cespare/xxhash v1.1.0
github.com/cespare/xxhash/v2 v2.1.1 // indirect
github.com/chrislusf/raft v1.0.7
github.com/colinmarc/hdfs/v2 v2.2.0
github.com/cespare/xxhash/v2 v2.1.2 // indirect
github.com/chrislusf/raft v1.0.8
github.com/colinmarc/hdfs/v2 v2.3.0
github.com/coreos/go-semver v0.3.0 // indirect github.com/coreos/go-semver v0.3.0 // indirect
github.com/coreos/go-systemd/v22 v22.3.2 // indirect github.com/coreos/go-systemd/v22 v22.3.2 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect github.com/davecgh/go-spew v1.1.1 // indirect
@ -32,13 +32,13 @@ require (
github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 // indirect github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 // indirect
github.com/facebookgo/stats v0.0.0-20151006221625-1b76add642e4 github.com/facebookgo/stats v0.0.0-20151006221625-1b76add642e4
github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4 // indirect github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4 // indirect
github.com/fclairamb/ftpserverlib v0.8.0
github.com/fclairamb/ftpserverlib v0.17.0
github.com/frankban/quicktest v1.7.2 // indirect github.com/frankban/quicktest v1.7.2 // indirect
github.com/fsnotify/fsnotify v1.4.9 // indirect
github.com/fsnotify/fsnotify v1.5.1 // indirect
github.com/go-errors/errors v1.1.1 // indirect github.com/go-errors/errors v1.1.1 // indirect
github.com/go-redis/redis/v8 v8.4.4
github.com/go-redis/redis/v8 v8.11.5
github.com/go-redsync/redsync/v4 v4.4.1 github.com/go-redsync/redsync/v4 v4.4.1
github.com/go-sql-driver/mysql v1.5.0
github.com/go-sql-driver/mysql v1.6.0
github.com/go-stack/stack v1.8.0 // indirect github.com/go-stack/stack v1.8.0 // indirect
github.com/go-zookeeper/zk v1.0.2 // indirect github.com/go-zookeeper/zk v1.0.2 // indirect
github.com/gocql/gocql v0.0.0-20210707082121-9a3953d1826d github.com/gocql/gocql v0.0.0-20210707082121-9a3953d1826d
@ -46,12 +46,11 @@ require (
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/golang/protobuf v1.5.2 github.com/golang/protobuf v1.5.2
github.com/golang/snappy v0.0.4 // indirect github.com/golang/snappy v0.0.4 // indirect
github.com/google/btree v1.0.0
github.com/google/go-cmp v0.5.6 // indirect
github.com/google/btree v1.0.1
github.com/google/go-cmp v0.5.7 // indirect
github.com/google/uuid v1.3.0 github.com/google/uuid v1.3.0
github.com/google/wire v0.4.0 // indirect
github.com/googleapis/gax-go v2.0.2+incompatible // indirect
github.com/googleapis/gax-go/v2 v2.1.0 // indirect
github.com/google/wire v0.5.0 // indirect
github.com/googleapis/gax-go/v2 v2.1.1 // indirect
github.com/gorilla/mux v1.7.4 github.com/gorilla/mux v1.7.4
github.com/grpc-ecosystem/go-grpc-middleware v1.1.0 github.com/grpc-ecosystem/go-grpc-middleware v1.1.0
github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed // indirect github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed // indirect
@ -60,34 +59,29 @@ require (
github.com/hashicorp/go-uuid v1.0.2 // indirect github.com/hashicorp/go-uuid v1.0.2 // indirect
github.com/hashicorp/hcl v1.0.0 // indirect github.com/hashicorp/hcl v1.0.0 // indirect
github.com/jcmturner/gofork v1.0.0 // indirect github.com/jcmturner/gofork v1.0.0 // indirect
github.com/jcmturner/gokrb5/v8 v8.4.1
github.com/jinzhu/copier v0.2.8
github.com/jcmturner/gokrb5/v8 v8.4.2
github.com/jinzhu/copier v0.3.5
github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect
github.com/json-iterator/go v1.1.11
github.com/karlseguin/ccache/v2 v2.0.7
github.com/json-iterator/go v1.1.12
github.com/karlseguin/ccache/v2 v2.0.8
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
github.com/klauspost/compress v1.13.6 // indirect github.com/klauspost/compress v1.13.6 // indirect
github.com/klauspost/cpuid v1.2.1 // indirect github.com/klauspost/cpuid v1.2.1 // indirect
github.com/klauspost/reedsolomon v1.9.2 github.com/klauspost/reedsolomon v1.9.2
github.com/konsorten/go-windows-terminal-sequences v1.0.3 // indirect
github.com/kurin/blazer v0.5.3 github.com/kurin/blazer v0.5.3
github.com/lib/pq v1.10.0
github.com/lib/pq v1.10.4
github.com/linxGnu/grocksdb v1.6.38 github.com/linxGnu/grocksdb v1.6.38
github.com/magiconair/properties v1.8.1 // indirect github.com/magiconair/properties v1.8.1 // indirect
github.com/mailru/easyjson v0.7.1 // indirect github.com/mailru/easyjson v0.7.1 // indirect
github.com/mattn/go-ieproxy v0.0.1 // indirect github.com/mattn/go-ieproxy v0.0.1 // indirect
github.com/mattn/go-isatty v0.0.12 // indirect github.com/mattn/go-isatty v0.0.12 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect
github.com/mitchellh/mapstructure v1.1.2 // indirect
github.com/mitchellh/mapstructure v1.4.1 // indirect
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.1 // indirect
github.com/nats-io/jwt v1.0.1 // indirect
github.com/nats-io/nats.go v1.10.0 // indirect
github.com/nats-io/nkeys v0.2.0 // indirect
github.com/nats-io/nuid v1.0.1 // indirect
github.com/modern-go/reflect2 v1.0.2 // indirect
github.com/olivere/elastic/v7 v7.0.19 github.com/olivere/elastic/v7 v7.0.19
github.com/pelletier/go-toml v1.7.0 // indirect github.com/pelletier/go-toml v1.7.0 // indirect
github.com/peterh/liner v1.1.0
github.com/peterh/liner v1.2.2
github.com/pierrec/lz4 v2.2.7+incompatible // indirect github.com/pierrec/lz4 v2.2.7+incompatible // indirect
github.com/pkg/errors v0.9.1 // indirect github.com/pkg/errors v0.9.1 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect
@ -99,18 +93,18 @@ require (
github.com/prometheus/procfs v0.6.0 // indirect github.com/prometheus/procfs v0.6.0 // indirect
github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563 // indirect github.com/rcrowley/go-metrics v0.0.0-20190826022208-cac0b30c2563 // indirect
github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 // indirect github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 // indirect
github.com/seaweedfs/goexif v1.0.2
github.com/sirupsen/logrus v1.6.0 // indirect
github.com/seaweedfs/goexif v2.0.0+incompatible
github.com/sirupsen/logrus v1.8.1 // indirect
github.com/spaolacci/murmur3 v1.1.0 // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect
github.com/spf13/afero v1.6.0 // indirect
github.com/spf13/afero v1.7.0 // indirect
github.com/spf13/cast v1.3.0 // indirect github.com/spf13/cast v1.3.0 // indirect
github.com/spf13/jwalterweatherman v1.1.0 // indirect github.com/spf13/jwalterweatherman v1.1.0 // indirect
github.com/spf13/viper v1.4.0 github.com/spf13/viper v1.4.0
github.com/streadway/amqp v0.0.0-20200108173154-1c71cc93ed71
github.com/stretchr/testify v1.7.0
github.com/streadway/amqp v1.0.0
github.com/stretchr/testify v1.7.1
github.com/stvp/tempredis v0.0.0-20181119212430-b82af8480203 github.com/stvp/tempredis v0.0.0-20181119212430-b82af8480203
github.com/syndtr/goleveldb v1.0.1-0.20190318030020-c3a204f8e965 github.com/syndtr/goleveldb v1.0.1-0.20190318030020-c3a204f8e965
github.com/tidwall/gjson v1.10.2
github.com/tidwall/gjson v1.14.0
github.com/tidwall/match v1.1.1 github.com/tidwall/match v1.1.1
github.com/tidwall/pretty v1.2.0 // indirect github.com/tidwall/pretty v1.2.0 // indirect
github.com/tsuna/gohbase v0.0.0-20201125011725-348991136365 github.com/tsuna/gohbase v0.0.0-20201125011725-348991136365
@ -124,25 +118,24 @@ require (
github.com/xdg-go/stringprep v1.0.2 // indirect github.com/xdg-go/stringprep v1.0.2 // indirect
github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d // indirect
go.etcd.io/etcd/client/v3 v3.5.0 go.etcd.io/etcd/client/v3 v3.5.0
go.mongodb.org/mongo-driver v1.8.0
go.mongodb.org/mongo-driver v1.8.4
go.opencensus.io v0.23.0 // indirect go.opencensus.io v0.23.0 // indirect
go.opentelemetry.io/otel v0.15.0 // indirect
gocloud.dev v0.20.0
gocloud.dev v0.24.0
gocloud.dev/pubsub/natspubsub v0.20.0 gocloud.dev/pubsub/natspubsub v0.20.0
gocloud.dev/pubsub/rabbitpubsub v0.20.0
golang.org/x/crypto v0.0.0-20201216223049-8b5274cf687f // indirect
gocloud.dev/pubsub/rabbitpubsub v0.24.0
golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa // indirect
golang.org/x/image v0.0.0-20200119044424-58c23975cae1 golang.org/x/image v0.0.0-20200119044424-58c23975cae1
golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d
golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f // indirect
golang.org/x/sys v0.0.0-20220111092808-5a964db01320
golang.org/x/net v0.0.0-20220225172249-27dd8689420f
golang.org/x/oauth2 v0.0.0-20220309155454-6242fa91716a // indirect
golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5
golang.org/x/text v0.3.7 // indirect golang.org/x/text v0.3.7 // indirect
golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2 golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
google.golang.org/api v0.57.0
google.golang.org/api v0.73.0
google.golang.org/appengine v1.6.7 // indirect google.golang.org/appengine v1.6.7 // indirect
google.golang.org/genproto v0.0.0-20210921142501-181ce0d877f6 // indirect
google.golang.org/grpc v1.40.0
google.golang.org/protobuf v1.27.1
google.golang.org/genproto v0.0.0-20220310185008-1973136f34c6 // indirect
google.golang.org/grpc v1.45.0
google.golang.org/protobuf v1.28.0
gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/inf.v0 v0.9.1 // indirect
gopkg.in/jcmturner/aescts.v1 v1.0.1 // indirect gopkg.in/jcmturner/aescts.v1 v1.0.1 // indirect
gopkg.in/jcmturner/dnsutils.v1 v1.0.1 // indirect gopkg.in/jcmturner/dnsutils.v1 v1.0.1 // indirect
@ -150,14 +143,14 @@ require (
gopkg.in/jcmturner/gokrb5.v7 v7.3.0 // indirect gopkg.in/jcmturner/gokrb5.v7 v7.3.0 // indirect
gopkg.in/jcmturner/rpc.v1 v1.1.0 // indirect gopkg.in/jcmturner/rpc.v1 v1.1.0 // indirect
modernc.org/b v1.0.0 // indirect modernc.org/b v1.0.0 // indirect
modernc.org/cc/v3 v3.33.5 // indirect
modernc.org/ccgo/v3 v3.9.4 // indirect
modernc.org/libc v1.9.5 // indirect
modernc.org/mathutil v1.2.2 // indirect
modernc.org/memory v1.0.4 // indirect
modernc.org/cc/v3 v3.35.24 // indirect
modernc.org/ccgo/v3 v3.15.17 // indirect
modernc.org/libc v1.14.12 // indirect
modernc.org/mathutil v1.4.1 // indirect
modernc.org/memory v1.0.7 // indirect
modernc.org/opt v0.1.1 // indirect modernc.org/opt v0.1.1 // indirect
modernc.org/sqlite v1.10.7
modernc.org/strutil v1.1.0 // indirect
modernc.org/sqlite v1.15.3
modernc.org/strutil v1.1.1 // indirect
modernc.org/token v1.0.0 // indirect modernc.org/token v1.0.0 // indirect
) )
@ -167,30 +160,47 @@ require (
) )
require ( require (
cloud.google.com/go/kms v1.0.0 // indirect
cloud.google.com/go/compute v1.5.0 // indirect
cloud.google.com/go/iam v0.1.1 // indirect
github.com/DataDog/zstd v1.3.6-0.20190409195224-796139022798 // indirect github.com/DataDog/zstd v1.3.6-0.20190409195224-796139022798 // indirect
github.com/arangodb/go-driver v1.2.1 // indirect github.com/arangodb/go-driver v1.2.1 // indirect
github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e // indirect github.com/arangodb/go-velocypack v0.0.0-20200318135517-5af53c29c67e // indirect
github.com/aws/aws-sdk-go-v2 v1.9.0 // indirect
github.com/aws/aws-sdk-go-v2/config v1.7.0 // indirect
github.com/aws/aws-sdk-go-v2/credentials v1.4.0 // indirect
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.5.0 // indirect
github.com/aws/aws-sdk-go-v2/internal/ini v1.2.2 // indirect
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.3.0 // indirect
github.com/aws/aws-sdk-go-v2/service/sso v1.4.0 // indirect
github.com/aws/aws-sdk-go-v2/service/sts v1.7.0 // indirect
github.com/aws/smithy-go v1.8.0 // indirect
github.com/d4l3k/messagediff v1.2.1 // indirect github.com/d4l3k/messagediff v1.2.1 // indirect
github.com/fclairamb/go-log v0.1.0 // indirect
github.com/gogo/protobuf v1.3.2 // indirect github.com/gogo/protobuf v1.3.2 // indirect
github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8 // indirect
github.com/jcmturner/aescts/v2 v2.0.0 // indirect github.com/jcmturner/aescts/v2 v2.0.0 // indirect
github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect
github.com/jcmturner/goidentity/v6 v6.0.1 // indirect github.com/jcmturner/goidentity/v6 v6.0.1 // indirect
github.com/jcmturner/rpc/v2 v2.0.2 // indirect
github.com/jcmturner/rpc/v2 v2.0.3 // indirect
github.com/mattn/go-runewidth v0.0.7 // indirect github.com/mattn/go-runewidth v0.0.7 // indirect
github.com/mattn/go-sqlite3 v2.0.1+incompatible // indirect github.com/mattn/go-sqlite3 v2.0.1+incompatible // indirect
github.com/nats-io/nats.go v1.11.0 // indirect
github.com/nats-io/nkeys v0.3.0 // indirect
github.com/nats-io/nuid v1.0.1 // indirect
github.com/philhofer/fwd v1.1.1 // indirect github.com/philhofer/fwd v1.1.1 // indirect
github.com/rwcarlsen/goexif v0.0.0-20190401172101-9e8deecbddbd // indirect
github.com/spf13/pflag v1.0.5 // indirect github.com/spf13/pflag v1.0.5 // indirect
github.com/tinylib/msgp v1.1.6 // indirect github.com/tinylib/msgp v1.1.6 // indirect
go.etcd.io/etcd/api/v3 v3.5.0 // indirect go.etcd.io/etcd/api/v3 v3.5.0 // indirect
go.etcd.io/etcd/client/pkg/v3 v3.5.0 // indirect go.etcd.io/etcd/client/pkg/v3 v3.5.0 // indirect
go.uber.org/atomic v1.7.0 // indirect
go.uber.org/atomic v1.9.0 // indirect
go.uber.org/multierr v1.7.0 // indirect go.uber.org/multierr v1.7.0 // indirect
go.uber.org/zap v1.17.0 // indirect
golang.org/x/mod v0.4.2 // indirect
go.uber.org/zap v1.19.0 // indirect
golang.org/x/mod v0.5.0 // indirect
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
lukechampine.com/uint128 v1.1.1 // indirect
) )
// replace github.com/chrislusf/raft => /Users/chrislu/go/src/github.com/chrislusf/raft // replace github.com/chrislusf/raft => /Users/chrislu/go/src/github.com/chrislusf/raft

727
go.sum
File diff suppressed because it is too large
View File

4
k8s/helm_charts2/Chart.yaml

@ -1,5 +1,5 @@
apiVersion: v1 apiVersion: v1
description: SeaweedFS description: SeaweedFS
name: seaweedfs name: seaweedfs
appVersion: "2.94"
version: "2.94"
appVersion: "2.96"
version: "2.96"

2
weed/command/benchmark.go

@ -129,7 +129,7 @@ func runBenchmark(cmd *Command, args []string) bool {
defer pprof.StopCPUProfile() defer pprof.StopCPUProfile()
} }
b.masterClient = wdclient.NewMasterClient(b.grpcDialOption, "client", "", "", pb.ServerAddresses(*b.masters).ToAddresses())
b.masterClient = wdclient.NewMasterClient(b.grpcDialOption, "client", "", "", pb.ServerAddresses(*b.masters).ToAddressMap())
go b.masterClient.KeepConnectedToMaster() go b.masterClient.KeepConnectedToMaster()
b.masterClient.WaitUntilConnected() b.masterClient.WaitUntilConnected()

7
weed/command/filer.go

@ -13,7 +13,7 @@ import (
"github.com/chrislusf/seaweedfs/weed/pb" "github.com/chrislusf/seaweedfs/weed/pb"
"github.com/chrislusf/seaweedfs/weed/pb/filer_pb" "github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
"github.com/chrislusf/seaweedfs/weed/security" "github.com/chrislusf/seaweedfs/weed/security"
"github.com/chrislusf/seaweedfs/weed/server"
weed_server "github.com/chrislusf/seaweedfs/weed/server"
stats_collect "github.com/chrislusf/seaweedfs/weed/stats" stats_collect "github.com/chrislusf/seaweedfs/weed/stats"
"github.com/chrislusf/seaweedfs/weed/util" "github.com/chrislusf/seaweedfs/weed/util"
) )
@ -29,7 +29,7 @@ var (
) )
type FilerOptions struct { type FilerOptions struct {
masters []pb.ServerAddress
masters map[string]pb.ServerAddress
mastersString *string mastersString *string
ip *string ip *string
bindIp *string bindIp *string
@ -103,6 +103,7 @@ func init() {
// start iam on filer // start iam on filer
filerStartIam = cmdFiler.Flag.Bool("iam", false, "whether to start IAM service") filerStartIam = cmdFiler.Flag.Bool("iam", false, "whether to start IAM service")
filerIamOptions.ip = cmdFiler.Flag.String("iam.ip", *f.ip, "iam server http listen ip address")
filerIamOptions.port = cmdFiler.Flag.Int("iam.port", 8111, "iam server http listen port") filerIamOptions.port = cmdFiler.Flag.Int("iam.port", 8111, "iam server http listen port")
} }
@ -170,7 +171,7 @@ func runFiler(cmd *Command, args []string) bool {
}() }()
} }
f.masters = pb.ServerAddresses(*f.mastersString).ToAddresses()
f.masters = pb.ServerAddresses(*f.mastersString).ToAddressMap()
f.startFiler() f.startFiler()

2
weed/command/iam.go

@ -67,7 +67,7 @@ func (iamopt *IamOptions) startIamServer() bool {
} }
} }
masters := pb.ServerAddresses(*iamopt.masters).ToAddresses()
masters := pb.ServerAddresses(*iamopt.masters).ToAddressMap()
router := mux.NewRouter().SkipClean(true) router := mux.NewRouter().SkipClean(true)
_, iamApiServer_err := iamapi.NewIamApiServer(router, &iamapi.IamServerOption{ _, iamApiServer_err := iamapi.NewIamApiServer(router, &iamapi.IamServerOption{
Masters: masters, Masters: masters,

11
weed/command/master.go

@ -127,13 +127,18 @@ func startMaster(masterOption MasterOptions, masterWhiteList []string) {
*masterOption.portGrpc = 10000 + *masterOption.port *masterOption.portGrpc = 10000 + *masterOption.port
} }
if *masterOption.ipBind == "" { if *masterOption.ipBind == "" {
*masterOption.ipBind = "localhost"
*masterOption.ipBind = *masterOption.ip
} }
myMasterAddress, peers := checkPeers(*masterOption.ip, *masterOption.port, *masterOption.portGrpc, *masterOption.peers) myMasterAddress, peers := checkPeers(*masterOption.ip, *masterOption.port, *masterOption.portGrpc, *masterOption.peers)
masterPeers := make(map[string]pb.ServerAddress)
for _, peer := range peers {
masterPeers[peer.String()] = peer
}
r := mux.NewRouter() r := mux.NewRouter()
ms := weed_server.NewMasterServer(r, masterOption.toMasterOption(masterWhiteList), peers)
ms := weed_server.NewMasterServer(r, masterOption.toMasterOption(masterWhiteList), masterPeers)
listeningAddress := util.JoinHostPort(*masterOption.ipBind, *masterOption.port) listeningAddress := util.JoinHostPort(*masterOption.ipBind, *masterOption.port)
glog.V(0).Infof("Start Seaweed Master %s at %s", util.Version(), listeningAddress) glog.V(0).Infof("Start Seaweed Master %s at %s", util.Version(), listeningAddress)
masterListener, masterLocalListner, e := util.NewIpAndLocalListeners(*masterOption.ipBind, *masterOption.port, 0) masterListener, masterLocalListner, e := util.NewIpAndLocalListeners(*masterOption.ipBind, *masterOption.port, 0)
@ -144,7 +149,7 @@ func startMaster(masterOption MasterOptions, masterWhiteList []string) {
// start raftServer // start raftServer
raftServerOption := &weed_server.RaftServerOption{ raftServerOption := &weed_server.RaftServerOption{
GrpcDialOption: security.LoadClientTLS(util.GetViper(), "grpc.master"), GrpcDialOption: security.LoadClientTLS(util.GetViper(), "grpc.master"),
Peers: peers,
Peers: masterPeers,
ServerAddr: myMasterAddress, ServerAddr: myMasterAddress,
DataDir: util.ResolvePath(*masterOption.metaFolder), DataDir: util.ResolvePath(*masterOption.metaFolder),
Topo: ms.Topo, Topo: ms.Topo,

15
weed/command/master_follower.go

@ -3,17 +3,18 @@ package command
import ( import (
"context" "context"
"fmt" "fmt"
"net/http"
"time"
"github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws"
"github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/glog"
"github.com/chrislusf/seaweedfs/weed/pb" "github.com/chrislusf/seaweedfs/weed/pb"
"github.com/chrislusf/seaweedfs/weed/pb/master_pb" "github.com/chrislusf/seaweedfs/weed/pb/master_pb"
"github.com/chrislusf/seaweedfs/weed/security" "github.com/chrislusf/seaweedfs/weed/security"
"github.com/chrislusf/seaweedfs/weed/server"
weed_server "github.com/chrislusf/seaweedfs/weed/server"
"github.com/chrislusf/seaweedfs/weed/util" "github.com/chrislusf/seaweedfs/weed/util"
"github.com/gorilla/mux" "github.com/gorilla/mux"
"google.golang.org/grpc/reflection" "google.golang.org/grpc/reflection"
"net/http"
"time"
) )
var ( var (
@ -45,13 +46,13 @@ var cmdMasterFollower = &Command{
Short: "start a master follower", Short: "start a master follower",
Long: `start a master follower to provide volume=>location mapping service Long: `start a master follower to provide volume=>location mapping service
The master follower does not participate in master election.
The master follower does not participate in master election.
It just follow the existing masters, and listen for any volume location changes. It just follow the existing masters, and listen for any volume location changes.
In most cases, the master follower is not needed. In big data centers with thousands of volume In most cases, the master follower is not needed. In big data centers with thousands of volume
servers. In theory, the master may have trouble to keep up with the write requests and read requests. servers. In theory, the master may have trouble to keep up with the write requests and read requests.
The master follower can relieve the master from from read requests, which only needs to
The master follower can relieve the master from from read requests, which only needs to
lookup a fileId or volumeId. lookup a fileId or volumeId.
The master follower currently can handle fileId lookup requests: The master follower currently can handle fileId lookup requests:
@ -82,7 +83,7 @@ func runMasterFollower(cmd *Command, args []string) bool {
func startMasterFollower(masterOptions MasterOptions) { func startMasterFollower(masterOptions MasterOptions) {
// collect settings from main masters // collect settings from main masters
masters := pb.ServerAddresses(*mf.peers).ToAddresses()
masters := pb.ServerAddresses(*mf.peers).ToAddressMap()
var err error var err error
grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.master") grpcDialOption := security.LoadClientTLS(util.GetViper(), "grpc.master")
@ -112,7 +113,7 @@ func startMasterFollower(masterOptions MasterOptions) {
option.IsFollower = true option.IsFollower = true
if *masterOptions.ipBind == "" { if *masterOptions.ipBind == "" {
*masterOptions.ipBind = "localhost"
*masterOptions.ipBind = *masterOptions.ip
} }
r := mux.NewRouter() r := mux.NewRouter()

2
weed/command/server.go

@ -191,7 +191,7 @@ func runServer(cmd *Command, args []string) bool {
// ip address // ip address
masterOptions.ip = serverIp masterOptions.ip = serverIp
masterOptions.ipBind = serverBindIp masterOptions.ipBind = serverBindIp
filerOptions.masters = pb.ServerAddresses(*masterOptions.peers).ToAddresses()
filerOptions.masters = pb.ServerAddresses(*masterOptions.peers).ToAddressMap()
filerOptions.ip = serverIp filerOptions.ip = serverIp
filerOptions.bindIp = serverBindIp filerOptions.bindIp = serverBindIp
s3Options.bindIp = serverBindIp s3Options.bindIp = serverBindIp

8
weed/filer/filechunks.go

@ -23,7 +23,13 @@ func TotalSize(chunks []*filer_pb.FileChunk) (size uint64) {
} }
func FileSize(entry *filer_pb.Entry) (size uint64) { func FileSize(entry *filer_pb.Entry) (size uint64) {
return maxUint64(TotalSize(entry.Chunks), entry.Attributes.FileSize)
fileSize := entry.Attributes.FileSize
if entry.RemoteEntry != nil {
if entry.RemoteEntry.RemoteMtime > entry.Attributes.Mtime {
fileSize = maxUint64(fileSize, uint64(entry.RemoteEntry.RemoteSize))
}
}
return maxUint64(TotalSize(entry.Chunks), fileSize)
} }
func ETag(entry *filer_pb.Entry) (etag string) { func ETag(entry *filer_pb.Entry) (etag string) {

2
weed/filer/filer.go

@ -49,7 +49,7 @@ type Filer struct {
UniqueFileId uint32 UniqueFileId uint32
} }
func NewFiler(masters []pb.ServerAddress, grpcDialOption grpc.DialOption,
func NewFiler(masters map[string]pb.ServerAddress, grpcDialOption grpc.DialOption,
filerHost pb.ServerAddress, collection string, replication string, dataCenter string, notifyFn func()) *Filer { filerHost pb.ServerAddress, collection string, replication string, dataCenter string, notifyFn func()) *Filer {
f := &Filer{ f := &Filer{
MasterClient: wdclient.NewMasterClient(grpcDialOption, cluster.FilerType, filerHost, dataCenter, masters), MasterClient: wdclient.NewMasterClient(grpcDialOption, cluster.FilerType, filerHost, dataCenter, masters),

3
weed/filer/meta_aggregator.go

@ -76,9 +76,6 @@ func (ma *MetaAggregator) setActive(address pb.ServerAddress, isActive bool) (no
} }
} else { } else {
if _, found := ma.peerStatues[address]; found { if _, found := ma.peerStatues[address]; found {
ma.peerStatues[address] -= 1
}
if ma.peerStatues[address] <= 0 {
delete(ma.peerStatues, address) delete(ma.peerStatues, address)
} }
} }

1
weed/filer/read_write.go

@ -62,6 +62,7 @@ func SaveInsideFiler(client filer_pb.SeaweedFilerClient, dir, name string, conte
}, },
Content: content, Content: content,
}, },
SkipCheckParentDirectory: true,
}) })
} else if err == nil { } else if err == nil {
entry := resp.Entry entry := resp.Entry

2
weed/iamapi/iamapi_server.go

@ -33,7 +33,7 @@ type IamS3ApiConfigure struct {
} }
type IamServerOption struct { type IamServerOption struct {
Masters []pb.ServerAddress
Masters map[string]pb.ServerAddress
Filer pb.ServerAddress Filer pb.ServerAddress
Port int Port int
GrpcDialOption grpc.DialOption GrpcDialOption grpc.DialOption

7
weed/mount/weedfs_file_mkrm.go

@ -77,9 +77,10 @@ func (wfs *WFS) Mknod(cancel <-chan struct{}, in *fuse.MknodIn, name string, out
defer wfs.mapPbIdFromFilerToLocal(newEntry) defer wfs.mapPbIdFromFilerToLocal(newEntry)
request := &filer_pb.CreateEntryRequest{ request := &filer_pb.CreateEntryRequest{
Directory: string(dirFullPath),
Entry: newEntry,
Signatures: []int32{wfs.signature},
Directory: string(dirFullPath),
Entry: newEntry,
Signatures: []int32{wfs.signature},
SkipCheckParentDirectory: true,
} }
glog.V(1).Infof("mknod: %v", request) glog.V(1).Infof("mknod: %v", request)

7
weed/mount/weedfs_file_sync.go

@ -140,9 +140,10 @@ func (wfs *WFS) doFlush(fh *FileHandle, uid, gid uint32) fuse.Status {
} }
request := &filer_pb.CreateEntryRequest{ request := &filer_pb.CreateEntryRequest{
Directory: string(dir),
Entry: entry,
Signatures: []int32{wfs.signature},
Directory: string(dir),
Entry: entry,
Signatures: []int32{wfs.signature},
SkipCheckParentDirectory: true,
} }
glog.V(4).Infof("%s set chunks: %v", fileFullPath, len(entry.Chunks)) glog.V(4).Infof("%s set chunks: %v", fileFullPath, len(entry.Chunks))

2
weed/pb/grpc_client_server.go

@ -206,7 +206,7 @@ func WithMasterClient(streamingMode bool, master ServerAddress, grpcDialOption g
} }
func WithOneOfGrpcMasterClients(streamingMode bool, masterGrpcAddresses []ServerAddress, grpcDialOption grpc.DialOption, fn func(client master_pb.SeaweedClient) error) (err error) {
func WithOneOfGrpcMasterClients(streamingMode bool, masterGrpcAddresses map[string]ServerAddress, grpcDialOption grpc.DialOption, fn func(client master_pb.SeaweedClient) error) (err error) {
for _, masterGrpcAddress := range masterGrpcAddresses { for _, masterGrpcAddress := range masterGrpcAddresses {
err = WithGrpcClient(streamingMode, func(grpcConnection *grpc.ClientConn) error { err = WithGrpcClient(streamingMode, func(grpcConnection *grpc.ClientConn) error {

4
weed/pb/remote.proto

@ -62,6 +62,10 @@ message RemoteConf {
string storj_secret_key = 66; string storj_secret_key = 66;
string storj_endpoint = 67; string storj_endpoint = 67;
string contabo_access_key = 68;
string contabo_secret_key = 69;
string contabo_endpoint = 70;
string contabo_region = 71;
} }
message RemoteStorageMapping { message RemoteStorageMapping {

105
weed/pb/remote_pb/remote.pb.go

@ -1,7 +1,7 @@
// Code generated by protoc-gen-go. DO NOT EDIT. // Code generated by protoc-gen-go. DO NOT EDIT.
// versions: // versions:
// protoc-gen-go v1.26.0 // protoc-gen-go v1.26.0
// protoc v3.17.3
// protoc v3.6.1
// source: remote.proto // source: remote.proto
package remote_pb package remote_pb
@ -69,6 +69,10 @@ type RemoteConf struct {
StorjAccessKey string `protobuf:"bytes,65,opt,name=storj_access_key,json=storjAccessKey,proto3" json:"storj_access_key,omitempty"` StorjAccessKey string `protobuf:"bytes,65,opt,name=storj_access_key,json=storjAccessKey,proto3" json:"storj_access_key,omitempty"`
StorjSecretKey string `protobuf:"bytes,66,opt,name=storj_secret_key,json=storjSecretKey,proto3" json:"storj_secret_key,omitempty"` StorjSecretKey string `protobuf:"bytes,66,opt,name=storj_secret_key,json=storjSecretKey,proto3" json:"storj_secret_key,omitempty"`
StorjEndpoint string `protobuf:"bytes,67,opt,name=storj_endpoint,json=storjEndpoint,proto3" json:"storj_endpoint,omitempty"` StorjEndpoint string `protobuf:"bytes,67,opt,name=storj_endpoint,json=storjEndpoint,proto3" json:"storj_endpoint,omitempty"`
ContaboAccessKey string `protobuf:"bytes,68,opt,name=contabo_access_key,json=contaboAccessKey,proto3" json:"contabo_access_key,omitempty"`
ContaboSecretKey string `protobuf:"bytes,69,opt,name=contabo_secret_key,json=contaboSecretKey,proto3" json:"contabo_secret_key,omitempty"`
ContaboEndpoint string `protobuf:"bytes,70,opt,name=contabo_endpoint,json=contaboEndpoint,proto3" json:"contabo_endpoint,omitempty"`
ContaboRegion string `protobuf:"bytes,71,opt,name=contabo_region,json=contaboRegion,proto3" json:"contabo_region,omitempty"`
} }
func (x *RemoteConf) Reset() { func (x *RemoteConf) Reset() {
@ -390,6 +394,34 @@ func (x *RemoteConf) GetStorjEndpoint() string {
return "" return ""
} }
func (x *RemoteConf) GetContaboAccessKey() string {
if x != nil {
return x.ContaboAccessKey
}
return ""
}
func (x *RemoteConf) GetContaboSecretKey() string {
if x != nil {
return x.ContaboSecretKey
}
return ""
}
func (x *RemoteConf) GetContaboEndpoint() string {
if x != nil {
return x.ContaboEndpoint
}
return ""
}
func (x *RemoteConf) GetContaboRegion() string {
if x != nil {
return x.ContaboRegion
}
return ""
}
type RemoteStorageMapping struct { type RemoteStorageMapping struct {
state protoimpl.MessageState state protoimpl.MessageState
sizeCache protoimpl.SizeCache sizeCache protoimpl.SizeCache
@ -512,7 +544,7 @@ var File_remote_proto protoreflect.FileDescriptor
var file_remote_proto_rawDesc = []byte{ var file_remote_proto_rawDesc = []byte{
0x0a, 0x0c, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x09, 0x0a, 0x0c, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x09,
0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x5f, 0x70, 0x62, 0x22, 0xe2, 0x0d, 0x0a, 0x0a, 0x52, 0x65,
0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x5f, 0x70, 0x62, 0x22, 0x90, 0x0f, 0x0a, 0x0a, 0x52, 0x65,
0x6d, 0x6f, 0x74, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65,
0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x12, 0x0a, 0x04,
0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65,
@ -622,35 +654,46 @@ var file_remote_proto_rawDesc = []byte{
0x79, 0x18, 0x42, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x73, 0x74, 0x6f, 0x72, 0x6a, 0x53, 0x65, 0x79, 0x18, 0x42, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x73, 0x74, 0x6f, 0x72, 0x6a, 0x53, 0x65,
0x63, 0x72, 0x65, 0x74, 0x4b, 0x65, 0x79, 0x12, 0x25, 0x0a, 0x0e, 0x73, 0x74, 0x6f, 0x72, 0x6a, 0x63, 0x72, 0x65, 0x74, 0x4b, 0x65, 0x79, 0x12, 0x25, 0x0a, 0x0e, 0x73, 0x74, 0x6f, 0x72, 0x6a,
0x5f, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x43, 0x20, 0x01, 0x28, 0x09, 0x52, 0x5f, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x43, 0x20, 0x01, 0x28, 0x09, 0x52,
0x0d, 0x73, 0x74, 0x6f, 0x72, 0x6a, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x22, 0xff,
0x01, 0x0a, 0x14, 0x52, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65,
0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x12, 0x49, 0x0a, 0x08, 0x6d, 0x61, 0x70, 0x70, 0x69,
0x6e, 0x67, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2d, 0x2e, 0x72, 0x65, 0x6d, 0x6f,
0x74, 0x65, 0x5f, 0x70, 0x62, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x53, 0x74, 0x6f, 0x72,
0x61, 0x67, 0x65, 0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x2e, 0x4d, 0x61, 0x70, 0x70, 0x69,
0x6e, 0x67, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x08, 0x6d, 0x61, 0x70, 0x70, 0x69, 0x6e,
0x67, 0x73, 0x12, 0x3d, 0x0a, 0x1b, 0x70, 0x72, 0x69, 0x6d, 0x61, 0x72, 0x79, 0x5f, 0x62, 0x75,
0x63, 0x6b, 0x65, 0x74, 0x5f, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x5f, 0x6e, 0x61, 0x6d,
0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x18, 0x70, 0x72, 0x69, 0x6d, 0x61, 0x72, 0x79,
0x42, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x4e, 0x61, 0x6d,
0x65, 0x1a, 0x5d, 0x0a, 0x0d, 0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x73, 0x45, 0x6e, 0x74,
0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52,
0x03, 0x6b, 0x65, 0x79, 0x12, 0x36, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20,
0x01, 0x28, 0x0b, 0x32, 0x20, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x5f, 0x70, 0x62, 0x2e,
0x52, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x4c, 0x6f, 0x63,
0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01,
0x22, 0x57, 0x0a, 0x15, 0x52, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67,
0x65, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d,
0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x16, 0x0a,
0x06, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x62,
0x75, 0x63, 0x6b, 0x65, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x61, 0x74, 0x68, 0x18, 0x03, 0x20,
0x01, 0x28, 0x09, 0x52, 0x04, 0x70, 0x61, 0x74, 0x68, 0x42, 0x50, 0x0a, 0x10, 0x73, 0x65, 0x61,
0x77, 0x65, 0x65, 0x64, 0x66, 0x73, 0x2e, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x42, 0x0a, 0x46,
0x69, 0x6c, 0x65, 0x72, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x5a, 0x30, 0x67, 0x69, 0x74, 0x68, 0x75,
0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x63, 0x68, 0x72, 0x69, 0x73, 0x6c, 0x75, 0x73, 0x66, 0x2f,
0x73, 0x65, 0x61, 0x77, 0x65, 0x65, 0x64, 0x66, 0x73, 0x2f, 0x77, 0x65, 0x65, 0x64, 0x2f, 0x70,
0x62, 0x2f, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x5f, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f,
0x74, 0x6f, 0x33,
0x0d, 0x73, 0x74, 0x6f, 0x72, 0x6a, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x2c,
0x0a, 0x12, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x62, 0x6f, 0x5f, 0x61, 0x63, 0x63, 0x65, 0x73, 0x73,
0x5f, 0x6b, 0x65, 0x79, 0x18, 0x44, 0x20, 0x01, 0x28, 0x09, 0x52, 0x10, 0x63, 0x6f, 0x6e, 0x74,
0x61, 0x62, 0x6f, 0x41, 0x63, 0x63, 0x65, 0x73, 0x73, 0x4b, 0x65, 0x79, 0x12, 0x2c, 0x0a, 0x12,
0x63, 0x6f, 0x6e, 0x74, 0x61, 0x62, 0x6f, 0x5f, 0x73, 0x65, 0x63, 0x72, 0x65, 0x74, 0x5f, 0x6b,
0x65, 0x79, 0x18, 0x45, 0x20, 0x01, 0x28, 0x09, 0x52, 0x10, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x62,
0x6f, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x4b, 0x65, 0x79, 0x12, 0x29, 0x0a, 0x10, 0x63, 0x6f,
0x6e, 0x74, 0x61, 0x62, 0x6f, 0x5f, 0x65, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x18, 0x46,
0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x62, 0x6f, 0x45, 0x6e, 0x64,
0x70, 0x6f, 0x69, 0x6e, 0x74, 0x12, 0x25, 0x0a, 0x0e, 0x63, 0x6f, 0x6e, 0x74, 0x61, 0x62, 0x6f,
0x5f, 0x72, 0x65, 0x67, 0x69, 0x6f, 0x6e, 0x18, 0x47, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0d, 0x63,
0x6f, 0x6e, 0x74, 0x61, 0x62, 0x6f, 0x52, 0x65, 0x67, 0x69, 0x6f, 0x6e, 0x22, 0xff, 0x01, 0x0a,
0x14, 0x52, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x4d, 0x61,
0x70, 0x70, 0x69, 0x6e, 0x67, 0x12, 0x49, 0x0a, 0x08, 0x6d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67,
0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2d, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65,
0x5f, 0x70, 0x62, 0x2e, 0x52, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67,
0x65, 0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x2e, 0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67,
0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x08, 0x6d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x73,
0x12, 0x3d, 0x0a, 0x1b, 0x70, 0x72, 0x69, 0x6d, 0x61, 0x72, 0x79, 0x5f, 0x62, 0x75, 0x63, 0x6b,
0x65, 0x74, 0x5f, 0x73, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18,
0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x18, 0x70, 0x72, 0x69, 0x6d, 0x61, 0x72, 0x79, 0x42, 0x75,
0x63, 0x6b, 0x65, 0x74, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x1a,
0x5d, 0x0a, 0x0d, 0x4d, 0x61, 0x70, 0x70, 0x69, 0x6e, 0x67, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79,
0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b,
0x65, 0x79, 0x12, 0x36, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28,
0x0b, 0x32, 0x20, 0x2e, 0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x5f, 0x70, 0x62, 0x2e, 0x52, 0x65,
0x6d, 0x6f, 0x74, 0x65, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x4c, 0x6f, 0x63, 0x61, 0x74,
0x69, 0x6f, 0x6e, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x57,
0x0a, 0x15, 0x52, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x4c,
0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18,
0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x62,
0x75, 0x63, 0x6b, 0x65, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x62, 0x75, 0x63,
0x6b, 0x65, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x61, 0x74, 0x68, 0x18, 0x03, 0x20, 0x01, 0x28,
0x09, 0x52, 0x04, 0x70, 0x61, 0x74, 0x68, 0x42, 0x50, 0x0a, 0x10, 0x73, 0x65, 0x61, 0x77, 0x65,
0x65, 0x64, 0x66, 0x73, 0x2e, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x42, 0x0a, 0x46, 0x69, 0x6c,
0x65, 0x72, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x5a, 0x30, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e,
0x63, 0x6f, 0x6d, 0x2f, 0x63, 0x68, 0x72, 0x69, 0x73, 0x6c, 0x75, 0x73, 0x66, 0x2f, 0x73, 0x65,
0x61, 0x77, 0x65, 0x65, 0x64, 0x66, 0x73, 0x2f, 0x77, 0x65, 0x65, 0x64, 0x2f, 0x70, 0x62, 0x2f,
0x72, 0x65, 0x6d, 0x6f, 0x74, 0x65, 0x5f, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f,
0x33,
} }
var ( var (

15
weed/pb/server_address.go

@ -86,6 +86,14 @@ func (sa ServerAddresses) ToAddresses() (addresses []ServerAddress) {
return return
} }
func (sa ServerAddresses) ToAddressMap() (addresses map[string]ServerAddress) {
addresses = make(map[string]ServerAddress)
for _, address := range sa.ToAddresses() {
addresses[address.String()] = address
}
return
}
func (sa ServerAddresses) ToAddressStrings() (addresses []string) { func (sa ServerAddresses) ToAddressStrings() (addresses []string) {
parts := strings.Split(string(sa), ",") parts := strings.Split(string(sa), ",")
for _, address := range parts { for _, address := range parts {
@ -101,6 +109,13 @@ func ToAddressStrings(addresses []ServerAddress) []string {
} }
return strings return strings
} }
func ToAddressStringsFromMap(addresses map[string]ServerAddress) []string {
var strings []string
for _, addr := range addresses {
strings = append(strings, string(addr))
}
return strings
}
func FromAddressStrings(strings []string) []ServerAddress { func FromAddressStrings(strings []string) []ServerAddress {
var addresses []ServerAddress var addresses []ServerAddress
for _, addr := range strings { for _, addr := range strings {

50
weed/remote_storage/s3/contabo.go

@ -0,0 +1,50 @@
package s3
import (
"fmt"
"os"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/credentials"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/s3"
"github.com/chrislusf/seaweedfs/weed/pb/remote_pb"
"github.com/chrislusf/seaweedfs/weed/remote_storage"
"github.com/chrislusf/seaweedfs/weed/util"
)
func init() {
remote_storage.RemoteStorageClientMakers["contabo"] = new(ContaboRemoteStorageMaker)
}
type ContaboRemoteStorageMaker struct{}
func (s ContaboRemoteStorageMaker) HasBucket() bool {
return true
}
func (s ContaboRemoteStorageMaker) Make(conf *remote_pb.RemoteConf) (remote_storage.RemoteStorageClient, error) {
client := &s3RemoteStorageClient{
conf: conf,
}
accessKey := util.Nvl(conf.ContaboAccessKey, os.Getenv("ACCESS_KEY"))
secretKey := util.Nvl(conf.ContaboSecretKey, os.Getenv("SECRET_KEY"))
config := &aws.Config{
Endpoint: aws.String(conf.ContaboEndpoint),
Region: aws.String(conf.ContaboRegion),
S3ForcePathStyle: aws.Bool(true),
S3DisableContentMD5Validation: aws.Bool(true),
}
if accessKey != "" && secretKey != "" {
config.Credentials = credentials.NewStaticCredentials(accessKey, secretKey, "")
}
sess, err := session.NewSession(config)
if err != nil {
return nil, fmt.Errorf("create contabo session: %v", err)
}
sess.Handlers.Build.PushFront(skipSha256PayloadSigning)
client.conn = s3.New(sess)
return client, nil
}

30
weed/s3api/filer_multipart.go

@ -5,6 +5,7 @@ import (
"fmt" "fmt"
"github.com/chrislusf/seaweedfs/weed/s3api/s3err" "github.com/chrislusf/seaweedfs/weed/s3api/s3err"
"path/filepath" "path/filepath"
"sort"
"strconv" "strconv"
"strings" "strings"
"time" "time"
@ -62,10 +63,15 @@ type CompleteMultipartUploadResult struct {
s3.CompleteMultipartUploadOutput s3.CompleteMultipartUploadOutput
} }
func (s3a *S3ApiServer) completeMultipartUpload(input *s3.CompleteMultipartUploadInput) (output *CompleteMultipartUploadResult, code s3err.ErrorCode) {
func (s3a *S3ApiServer) completeMultipartUpload(input *s3.CompleteMultipartUploadInput, parts *CompleteMultipartUpload) (output *CompleteMultipartUploadResult, code s3err.ErrorCode) {
glog.V(2).Infof("completeMultipartUpload input %v", input) glog.V(2).Infof("completeMultipartUpload input %v", input)
completedParts := parts.Parts
sort.Slice(completedParts, func(i, j int) bool {
return completedParts[i].PartNumber < completedParts[j].PartNumber
})
uploadDirectory := s3a.genUploadsFolder(*input.Bucket) + "/" + *input.UploadId uploadDirectory := s3a.genUploadsFolder(*input.Bucket) + "/" + *input.UploadId
entries, _, err := s3a.list(uploadDirectory, "", "", false, maxPartsList) entries, _, err := s3a.list(uploadDirectory, "", "", false, maxPartsList)
@ -80,14 +86,16 @@ func (s3a *S3ApiServer) completeMultipartUpload(input *s3.CompleteMultipartUploa
return nil, s3err.ErrNoSuchUpload return nil, s3err.ErrNoSuchUpload
} }
mime := pentry.Attributes.Mime
var finalParts []*filer_pb.FileChunk var finalParts []*filer_pb.FileChunk
var offset int64 var offset int64
var mime string
for _, entry := range entries { for _, entry := range entries {
if strings.HasSuffix(entry.Name, ".part") && !entry.IsDirectory { if strings.HasSuffix(entry.Name, ".part") && !entry.IsDirectory {
if entry.Name == "0001.part" && entry.Attributes.Mime != "" {
mime = entry.Attributes.Mime
_, found := findByPartNumber(entry.Name, completedParts)
if !found {
continue
} }
for _, chunk := range entry.Chunks { for _, chunk := range entry.Chunks {
p := &filer_pb.FileChunk{ p := &filer_pb.FileChunk{
@ -156,6 +164,20 @@ func (s3a *S3ApiServer) completeMultipartUpload(input *s3.CompleteMultipartUploa
return return
} }
func findByPartNumber(fileName string, parts []CompletedPart) (etag string, found bool) {
partNumber, formatErr := strconv.Atoi(fileName[:4])
if formatErr != nil {
return
}
x := sort.Search(len(parts), func(i int) bool {
return parts[i].PartNumber >= partNumber
})
if parts[x].PartNumber != partNumber {
return
}
return parts[x].ETag, true
}
func (s3a *S3ApiServer) abortMultipartUpload(input *s3.AbortMultipartUploadInput) (output *s3.AbortMultipartUploadOutput, code s3err.ErrorCode) { func (s3a *S3ApiServer) abortMultipartUpload(input *s3.AbortMultipartUploadInput) (output *s3.AbortMultipartUploadOutput, code s3err.ErrorCode) {
glog.V(2).Infof("abortMultipartUpload input %v", input) glog.V(2).Infof("abortMultipartUpload input %v", input)

83
weed/s3api/filer_multipart_test.go

@ -4,6 +4,7 @@ import (
"github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/s3" "github.com/aws/aws-sdk-go/service/s3"
"github.com/chrislusf/seaweedfs/weed/s3api/s3err" "github.com/chrislusf/seaweedfs/weed/s3api/s3err"
"github.com/stretchr/testify/assert"
"testing" "testing"
"time" "time"
) )
@ -48,3 +49,85 @@ func TestListPartsResult(t *testing.T) {
} }
} }
func Test_findByPartNumber(t *testing.T) {
type args struct {
fileName string
parts []CompletedPart
}
parts := []CompletedPart{
CompletedPart{
ETag: "xxx",
PartNumber: 1,
},
CompletedPart{
ETag: "yyy",
PartNumber: 3,
},
CompletedPart{
ETag: "zzz",
PartNumber: 5,
},
}
tests := []struct {
name string
args args
wantEtag string
wantFound bool
}{
{
"first",
args{
"0001.part",
parts,
},
"xxx",
true,
},
{
"second",
args{
"0002.part",
parts,
},
"",
false,
},
{
"third",
args{
"0003.part",
parts,
},
"yyy",
true,
},
{
"fourth",
args{
"0004.part",
parts,
},
"",
false,
},
{
"fifth",
args{
"0005.part",
parts,
},
"zzz",
true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
gotEtag, gotFound := findByPartNumber(tt.args.fileName, tt.args.parts)
assert.Equalf(t, tt.wantEtag, gotEtag, "findByPartNumber(%v, %v)", tt.args.fileName, tt.args.parts)
assert.Equalf(t, tt.wantFound, gotFound, "findByPartNumber(%v, %v)", tt.args.fileName, tt.args.parts)
})
}
}

9
weed/s3api/filer_util.go

@ -55,10 +55,11 @@ func (s3a *S3ApiServer) rm(parentDirectoryPath, entryName string, isDeleteData,
func doDeleteEntry(client filer_pb.SeaweedFilerClient, parentDirectoryPath string, entryName string, isDeleteData bool, isRecursive bool) error { func doDeleteEntry(client filer_pb.SeaweedFilerClient, parentDirectoryPath string, entryName string, isDeleteData bool, isRecursive bool) error {
request := &filer_pb.DeleteEntryRequest{ request := &filer_pb.DeleteEntryRequest{
Directory: parentDirectoryPath,
Name: entryName,
IsDeleteData: isDeleteData,
IsRecursive: isRecursive,
Directory: parentDirectoryPath,
Name: entryName,
IsDeleteData: isDeleteData,
IsRecursive: isRecursive,
IgnoreRecursiveError: true,
} }
glog.V(1).Infof("delete entry %v/%v: %v", parentDirectoryPath, entryName, request) glog.V(1).Infof("delete entry %v/%v: %v", parentDirectoryPath, entryName, request)

36
weed/s3api/s3api_object_multipart_handlers.go

@ -1,11 +1,13 @@
package s3api package s3api
import ( import (
"encoding/xml"
"fmt" "fmt"
"github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/glog"
xhttp "github.com/chrislusf/seaweedfs/weed/s3api/http" xhttp "github.com/chrislusf/seaweedfs/weed/s3api/http"
"github.com/chrislusf/seaweedfs/weed/s3api/s3err" "github.com/chrislusf/seaweedfs/weed/s3api/s3err"
weed_server "github.com/chrislusf/seaweedfs/weed/server" weed_server "github.com/chrislusf/seaweedfs/weed/server"
"io"
"net/http" "net/http"
"net/url" "net/url"
"strconv" "strconv"
@ -56,8 +58,16 @@ func (s3a *S3ApiServer) NewMultipartUploadHandler(w http.ResponseWriter, r *http
// CompleteMultipartUploadHandler - Completes multipart upload. // CompleteMultipartUploadHandler - Completes multipart upload.
func (s3a *S3ApiServer) CompleteMultipartUploadHandler(w http.ResponseWriter, r *http.Request) { func (s3a *S3ApiServer) CompleteMultipartUploadHandler(w http.ResponseWriter, r *http.Request) {
// https://docs.aws.amazon.com/AmazonS3/latest/API/API_CompleteMultipartUpload.html
bucket, object := xhttp.GetBucketAndObject(r) bucket, object := xhttp.GetBucketAndObject(r)
parts := &CompleteMultipartUpload{}
if err := xmlDecoder(r.Body, parts, r.ContentLength); err != nil {
s3err.WriteErrorResponse(w, r, s3err.ErrMalformedXML)
return
}
// Get upload id. // Get upload id.
uploadID, _, _, _ := getObjectResources(r.URL.Query()) uploadID, _, _, _ := getObjectResources(r.URL.Query())
@ -65,7 +75,7 @@ func (s3a *S3ApiServer) CompleteMultipartUploadHandler(w http.ResponseWriter, r
Bucket: aws.String(bucket), Bucket: aws.String(bucket),
Key: objectKey(aws.String(object)), Key: objectKey(aws.String(object)),
UploadId: aws.String(uploadID), UploadId: aws.String(uploadID),
})
}, parts)
glog.V(2).Info("CompleteMultipartUploadHandler", string(s3err.EncodeXMLResponse(response)), errCode) glog.V(2).Info("CompleteMultipartUploadHandler", string(s3err.EncodeXMLResponse(response)), errCode)
@ -268,8 +278,24 @@ func getObjectResources(values url.Values) (uploadID string, partNumberMarker, m
return return
} }
type byCompletedPartNumber []*s3.CompletedPart
func xmlDecoder(body io.Reader, v interface{}, size int64) error {
var lbody io.Reader
if size > 0 {
lbody = io.LimitReader(body, size)
} else {
lbody = body
}
d := xml.NewDecoder(lbody)
d.CharsetReader = func(label string, input io.Reader) (io.Reader, error) {
return input, nil
}
return d.Decode(v)
}
func (a byCompletedPartNumber) Len() int { return len(a) }
func (a byCompletedPartNumber) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
func (a byCompletedPartNumber) Less(i, j int) bool { return *a[i].PartNumber < *a[j].PartNumber }
type CompleteMultipartUpload struct {
Parts []CompletedPart `xml:"Part"`
}
type CompletedPart struct {
ETag string
PartNumber int
}

2
weed/server/filer_grpc_server.go

@ -393,7 +393,7 @@ func (fs *FilerServer) GetFilerConfiguration(ctx context.Context, req *filer_pb.
clusterId, _ := fs.filer.Store.KvGet(context.Background(), []byte("clusterId")) clusterId, _ := fs.filer.Store.KvGet(context.Background(), []byte("clusterId"))
t := &filer_pb.GetFilerConfigurationResponse{ t := &filer_pb.GetFilerConfigurationResponse{
Masters: pb.ToAddressStrings(fs.option.Masters),
Masters: pb.ToAddressStringsFromMap(fs.option.Masters),
Collection: fs.option.Collection, Collection: fs.option.Collection,
Replication: fs.option.DefaultReplication, Replication: fs.option.DefaultReplication,
MaxMb: uint32(fs.option.MaxMB), MaxMb: uint32(fs.option.MaxMB),

2
weed/server/filer_server.go

@ -49,7 +49,7 @@ import (
) )
type FilerOption struct { type FilerOption struct {
Masters []pb.ServerAddress
Masters map[string]pb.ServerAddress
Collection string Collection string
DefaultReplication string DefaultReplication string
DisableDirListing bool DisableDirListing bool

4
weed/server/filer_server_handlers_tagging.go

@ -82,7 +82,9 @@ func (fs *FilerServer) DeleteTaggingHandler(w http.ResponseWriter, r *http.Reque
toDelete := strings.Split(r.URL.Query().Get("tagging"), ",") toDelete := strings.Split(r.URL.Query().Get("tagging"), ",")
deletions := make(map[string]struct{}) deletions := make(map[string]struct{})
for _, deletion := range toDelete { for _, deletion := range toDelete {
deletions[deletion] = struct{}{}
if deletion != "" {
deletions[deletion] = struct{}{}
}
} }
// delete all tags or specific tags // delete all tags or specific tags

2
weed/server/filer_server_handlers_write_autochunk.go

@ -131,7 +131,7 @@ func isAppend(r *http.Request) bool {
} }
func skipCheckParentDirEntry(r *http.Request) bool { func skipCheckParentDirEntry(r *http.Request) bool {
return r.URL.Query().Get("skipCheckParentDir") != "true"
return r.URL.Query().Get("skipCheckParentDir") == "true"
} }
func (fs *FilerServer) saveMetaData(ctx context.Context, r *http.Request, fileName string, contentType string, so *operation.StorageOption, md5bytes []byte, fileChunks []*filer_pb.FileChunk, chunkOffset int64, content []byte) (filerResult *FilerPostResult, replyerr error) { func (fs *FilerServer) saveMetaData(ctx context.Context, r *http.Request, fileName string, contentType string, so *operation.StorageOption, md5bytes []byte, fileChunks []*filer_pb.FileChunk, chunkOffset int64, content []byte) (filerResult *FilerPostResult, replyerr error) {

2
weed/server/master_server.go

@ -75,7 +75,7 @@ type MasterServer struct {
Cluster *cluster.Cluster Cluster *cluster.Cluster
} }
func NewMasterServer(r *mux.Router, option *MasterOption, peers []pb.ServerAddress) *MasterServer {
func NewMasterServer(r *mux.Router, option *MasterOption, peers map[string]pb.ServerAddress) *MasterServer {
v := util.GetViper() v := util.GetViper()
signingKey := v.GetString("jwt.signing.key") signingKey := v.GetString("jwt.signing.key")

40
weed/server/raft_server.go

@ -5,8 +5,6 @@ import (
"math/rand" "math/rand"
"os" "os"
"path" "path"
"sort"
"strings"
"time" "time"
"google.golang.org/grpc" "google.golang.org/grpc"
@ -21,7 +19,7 @@ import (
type RaftServerOption struct { type RaftServerOption struct {
GrpcDialOption grpc.DialOption GrpcDialOption grpc.DialOption
Peers []pb.ServerAddress
Peers map[string]pb.ServerAddress
ServerAddr pb.ServerAddress ServerAddr pb.ServerAddress
DataDir string DataDir string
Topo *topology.Topology Topo *topology.Topology
@ -31,7 +29,7 @@ type RaftServerOption struct {
} }
type RaftServer struct { type RaftServer struct {
peers []pb.ServerAddress // initial peers to join with
peers map[string]pb.ServerAddress // initial peers to join with
raftServer raft.Server raftServer raft.Server
dataDir string dataDir string
serverAddr pb.ServerAddress serverAddr pb.ServerAddress
@ -81,10 +79,11 @@ func NewRaftServer(option *RaftServerOption) (*RaftServer, error) {
transporter := raft.NewGrpcTransporter(option.GrpcDialOption) transporter := raft.NewGrpcTransporter(option.GrpcDialOption)
glog.V(0).Infof("Starting RaftServer with %v", option.ServerAddr) glog.V(0).Infof("Starting RaftServer with %v", option.ServerAddr)
// always clear previous log to avoid server is promotable
os.RemoveAll(path.Join(s.dataDir, "log"))
if !option.RaftResumeState { if !option.RaftResumeState {
// always clear previous metadata // always clear previous metadata
os.RemoveAll(path.Join(s.dataDir, "conf")) os.RemoveAll(path.Join(s.dataDir, "conf"))
os.RemoveAll(path.Join(s.dataDir, "log"))
os.RemoveAll(path.Join(s.dataDir, "snapshot")) os.RemoveAll(path.Join(s.dataDir, "snapshot"))
} }
if err := os.MkdirAll(path.Join(s.dataDir, "snapshot"), 0600); err != nil { if err := os.MkdirAll(path.Join(s.dataDir, "snapshot"), 0600); err != nil {
@ -107,39 +106,26 @@ func NewRaftServer(option *RaftServerOption) (*RaftServer, error) {
return nil, err return nil, err
} }
for _, peer := range s.peers {
if err := s.raftServer.AddPeer(string(peer), peer.ToGrpcAddress()); err != nil {
for name, peer := range s.peers {
if err := s.raftServer.AddPeer(name, peer.ToGrpcAddress()); err != nil {
return nil, err return nil, err
} }
} }
// Remove deleted peers // Remove deleted peers
for existsPeerName := range s.raftServer.Peers() { for existsPeerName := range s.raftServer.Peers() {
exists := false
var existingPeer pb.ServerAddress
for _, peer := range s.peers {
if peer.ToGrpcAddress() == existsPeerName {
exists, existingPeer = true, peer
break
}
}
if exists {
if existingPeer, found := s.peers[existsPeerName]; !found {
if err := s.raftServer.RemovePeer(existsPeerName); err != nil { if err := s.raftServer.RemovePeer(existsPeerName); err != nil {
glog.V(0).Infoln(err) glog.V(0).Infoln(err)
return nil, err return nil, err
} else { } else {
glog.V(0).Infof("removing old peer %s", existingPeer)
glog.V(0).Infof("removing old peer: %s", existingPeer)
} }
} }
} }
s.GrpcServer = raft.NewGrpcServer(s.raftServer) s.GrpcServer = raft.NewGrpcServer(s.raftServer)
if s.raftServer.IsLogEmpty() && isTheFirstOne(option.ServerAddr, s.peers) {
// Initialize the server by joining itself.
// s.DoJoinCommand()
}
glog.V(0).Infof("current cluster leader: %v", s.raftServer.Leader()) glog.V(0).Infof("current cluster leader: %v", s.raftServer.Leader())
return s, nil return s, nil
@ -155,16 +141,6 @@ func (s *RaftServer) Peers() (members []string) {
return return
} }
func isTheFirstOne(self pb.ServerAddress, peers []pb.ServerAddress) bool {
sort.Slice(peers, func(i, j int) bool {
return strings.Compare(string(peers[i]), string(peers[j])) < 0
})
if len(peers) <= 0 {
return true
}
return self == peers[0]
}
func (s *RaftServer) DoJoinCommand() { func (s *RaftServer) DoJoinCommand() {
glog.V(0).Infoln("Initializing new cluster") glog.V(0).Infoln("Initializing new cluster")

2
weed/server/volume_server.go

@ -23,7 +23,6 @@ type VolumeServer struct {
inFlightDownloadDataSize int64 inFlightDownloadDataSize int64
concurrentUploadLimit int64 concurrentUploadLimit int64
concurrentDownloadLimit int64 concurrentDownloadLimit int64
inFlightUploadDataLimitCond *sync.Cond
inFlightDownloadDataLimitCond *sync.Cond inFlightDownloadDataLimitCond *sync.Cond
SeedMasterNodes []pb.ServerAddress SeedMasterNodes []pb.ServerAddress
@ -84,7 +83,6 @@ func NewVolumeServer(adminMux, publicMux *http.ServeMux, ip string,
fileSizeLimitBytes: int64(fileSizeLimitMB) * 1024 * 1024, fileSizeLimitBytes: int64(fileSizeLimitMB) * 1024 * 1024,
isHeartbeating: true, isHeartbeating: true,
stopChan: make(chan bool), stopChan: make(chan bool),
inFlightUploadDataLimitCond: sync.NewCond(new(sync.Mutex)),
inFlightDownloadDataLimitCond: sync.NewCond(new(sync.Mutex)), inFlightDownloadDataLimitCond: sync.NewCond(new(sync.Mutex)),
concurrentUploadLimit: concurrentUploadLimit, concurrentUploadLimit: concurrentUploadLimit,
concurrentDownloadLimit: concurrentDownloadLimit, concurrentDownloadLimit: concurrentDownloadLimit,

25
weed/server/volume_server_handlers.go

@ -1,6 +1,7 @@
package weed_server package weed_server
import ( import (
"fmt"
"net/http" "net/http"
"strconv" "strconv"
"strings" "strings"
@ -39,8 +40,14 @@ func (vs *VolumeServer) privateStoreHandler(w http.ResponseWriter, r *http.Reque
stats.ReadRequest() stats.ReadRequest()
vs.inFlightDownloadDataLimitCond.L.Lock() vs.inFlightDownloadDataLimitCond.L.Lock()
for vs.concurrentDownloadLimit != 0 && atomic.LoadInt64(&vs.inFlightDownloadDataSize) > vs.concurrentDownloadLimit { for vs.concurrentDownloadLimit != 0 && atomic.LoadInt64(&vs.inFlightDownloadDataSize) > vs.concurrentDownloadLimit {
glog.V(4).Infof("wait because inflight download data %d > %d", vs.inFlightDownloadDataSize, vs.concurrentDownloadLimit)
vs.inFlightDownloadDataLimitCond.Wait()
select {
case <-r.Context().Done():
glog.V(4).Infof("request cancelled from %s: %v", r.RemoteAddr, r.Context().Err())
return
default:
glog.V(4).Infof("wait because inflight download data %d > %d", vs.inFlightDownloadDataSize, vs.concurrentDownloadLimit)
vs.inFlightDownloadDataLimitCond.Wait()
}
} }
vs.inFlightDownloadDataLimitCond.L.Unlock() vs.inFlightDownloadDataLimitCond.L.Unlock()
vs.GetOrHeadHandler(w, r) vs.GetOrHeadHandler(w, r)
@ -51,16 +58,18 @@ func (vs *VolumeServer) privateStoreHandler(w http.ResponseWriter, r *http.Reque
// wait until in flight data is less than the limit // wait until in flight data is less than the limit
contentLength := getContentLength(r) contentLength := getContentLength(r)
vs.inFlightUploadDataLimitCond.L.Lock()
for vs.concurrentUploadLimit != 0 && atomic.LoadInt64(&vs.inFlightUploadDataSize) > vs.concurrentUploadLimit {
glog.V(4).Infof("wait because inflight upload data %d > %d", vs.inFlightUploadDataSize, vs.concurrentUploadLimit)
vs.inFlightUploadDataLimitCond.Wait()
// exclude the replication from the concurrentUploadLimitMB
if vs.concurrentUploadLimit != 0 && r.URL.Query().Get("type") != "replicate" &&
atomic.LoadInt64(&vs.inFlightUploadDataSize) > vs.concurrentUploadLimit {
err := fmt.Errorf("reject because inflight upload data %d > %d", vs.inFlightUploadDataSize, vs.concurrentUploadLimit)
glog.V(1).Infof("too many requests: %v", err)
writeJsonError(w, r, http.StatusTooManyRequests, err)
return
} }
vs.inFlightUploadDataLimitCond.L.Unlock()
atomic.AddInt64(&vs.inFlightUploadDataSize, contentLength) atomic.AddInt64(&vs.inFlightUploadDataSize, contentLength)
defer func() { defer func() {
atomic.AddInt64(&vs.inFlightUploadDataSize, -contentLength) atomic.AddInt64(&vs.inFlightUploadDataSize, -contentLength)
vs.inFlightUploadDataLimitCond.Signal()
}() }()
// processs uploads // processs uploads

2
weed/shell/commands.go

@ -46,7 +46,7 @@ var (
func NewCommandEnv(options *ShellOptions) *CommandEnv { func NewCommandEnv(options *ShellOptions) *CommandEnv {
ce := &CommandEnv{ ce := &CommandEnv{
env: make(map[string]string), env: make(map[string]string),
MasterClient: wdclient.NewMasterClient(options.GrpcDialOption, pb.AdminShellClient, "", "", pb.ServerAddresses(*options.Masters).ToAddresses()),
MasterClient: wdclient.NewMasterClient(options.GrpcDialOption, pb.AdminShellClient, "", "", pb.ServerAddresses(*options.Masters).ToAddressMap()),
option: options, option: options,
} }
ce.locker = exclusive_locks.NewExclusiveLocker(ce.MasterClient, "admin") ce.locker = exclusive_locks.NewExclusiveLocker(ce.MasterClient, "admin")

2
weed/storage/erasure_coding/ec_encoder.go

@ -220,7 +220,7 @@ func encodeDatFile(remainingSize int64, err error, baseFileName string, bufferSi
processedSize += largeBlockSize * DataShardsCount processedSize += largeBlockSize * DataShardsCount
} }
for remainingSize > 0 { for remainingSize > 0 {
encodeData(file, enc, processedSize, smallBlockSize, buffers, outputs)
err = encodeData(file, enc, processedSize, smallBlockSize, buffers, outputs)
if err != nil { if err != nil {
return fmt.Errorf("failed to encode small chunk data: %v", err) return fmt.Errorf("failed to encode small chunk data: %v", err)
} }

13
weed/topology/volume_layout.go

@ -140,9 +140,7 @@ func NewVolumeLayout(rp *super_block.ReplicaPlacement, ttl *needle.TTL, diskType
} }
func (vl *VolumeLayout) String() string { func (vl *VolumeLayout) String() string {
vl.accessLock.RLock()
defer vl.accessLock.RUnlock()
return fmt.Sprintf("rp:%v, ttl:%v, vid2location:%v, writables:%v, volumeSizeLimit:%v", vl.rp, vl.ttl, vl.vid2location, vl.writables, vl.volumeSizeLimit)
return fmt.Sprintf("rp:%v, ttl:%v, writables:%v, volumeSizeLimit:%v", vl.rp, vl.ttl, vl.writables, vl.volumeSizeLimit)
} }
func (vl *VolumeLayout) RegisterVolume(v *storage.VolumeInfo, dn *DataNode) { func (vl *VolumeLayout) RegisterVolume(v *storage.VolumeInfo, dn *DataNode) {
@ -220,6 +218,13 @@ func (vl *VolumeLayout) ensureCorrectWritables(vid needle.VolumeId) {
vl.setVolumeWritable(vid) vl.setVolumeWritable(vid)
} }
} else { } else {
if !vl.enoughCopies(vid) {
glog.V(0).Infof("volume %d does not have enough copies", vid)
}
if !vl.isAllWritable(vid) {
glog.V(0).Infof("volume %d are not all writable", vid)
}
glog.V(0).Infof("volume %d remove from writable", vid)
vl.removeFromWritable(vid) vl.removeFromWritable(vid)
} }
} }
@ -435,7 +440,7 @@ func (vl *VolumeLayout) SetVolumeCapacityFull(vid needle.VolumeId) bool {
vl.accessLock.Lock() vl.accessLock.Lock()
defer vl.accessLock.Unlock() defer vl.accessLock.Unlock()
// glog.V(0).Infoln("Volume", vid, "reaches full capacity.")
glog.V(0).Infof("Volume %d reaches full capacity.", vid)
return vl.removeFromWritable(vid) return vl.removeFromWritable(vid)
} }

25
weed/util/config.go

@ -9,6 +9,20 @@ import (
"github.com/chrislusf/seaweedfs/weed/glog" "github.com/chrislusf/seaweedfs/weed/glog"
) )
var (
ConfigurationFileDirectory DirectoryValueType
)
type DirectoryValueType string
func (s *DirectoryValueType) Set(value string) error {
*s = DirectoryValueType(value)
return nil
}
func (s *DirectoryValueType) String() string {
return string(*s)
}
type Configuration interface { type Configuration interface {
GetString(key string) string GetString(key string) string
GetBool(key string) bool GetBool(key string) bool
@ -20,11 +34,12 @@ type Configuration interface {
func LoadConfiguration(configFileName string, required bool) (loaded bool) { func LoadConfiguration(configFileName string, required bool) (loaded bool) {
// find a filer store // find a filer store
viper.SetConfigName(configFileName) // name of config file (without extension)
viper.AddConfigPath(".") // optionally look for config in the working directory
viper.AddConfigPath("$HOME/.seaweedfs") // call multiple times to add many search paths
viper.AddConfigPath("/usr/local/etc/seaweedfs/") // search path for bsd-style config directory in
viper.AddConfigPath("/etc/seaweedfs/") // path to look for the config file in
viper.SetConfigName(configFileName) // name of config file (without extension)
viper.AddConfigPath(ResolvePath(ConfigurationFileDirectory.String())) // path to look for the config file in
viper.AddConfigPath(".") // optionally look for config in the working directory
viper.AddConfigPath("$HOME/.seaweedfs") // call multiple times to add many search paths
viper.AddConfigPath("/usr/local/etc/seaweedfs/") // search path for bsd-style config directory in
viper.AddConfigPath("/etc/seaweedfs/") // path to look for the config file in
if err := viper.MergeInConfig(); err != nil { // Handle errors reading the config file if err := viper.MergeInConfig(); err != nil { // Handle errors reading the config file
if strings.Contains(err.Error(), "Not Found") { if strings.Contains(err.Error(), "Not Found") {

2
weed/util/constants.go

@ -5,7 +5,7 @@ import (
) )
var ( var (
VERSION_NUMBER = fmt.Sprintf("%.02f", 2.94)
VERSION_NUMBER = fmt.Sprintf("%.02f", 2.96)
VERSION = sizeLimit + " " + VERSION_NUMBER VERSION = sizeLimit + " " + VERSION_NUMBER
COMMIT = "" COMMIT = ""
) )

2
weed/util/net_timeout.go

@ -109,7 +109,7 @@ func NewIpAndLocalListeners(host string, port int, timeout time.Duration) (ipLis
WriteTimeout: timeout, WriteTimeout: timeout,
} }
if host != "localhost" && host != "" {
if host != "localhost" && host != "" && host != "0.0.0.0" && host != "127.0.0.1" {
listner, err = net.Listen("tcp", JoinHostPort("localhost", port)) listner, err = net.Listen("tcp", JoinHostPort("localhost", port))
if err != nil { if err != nil {
return return

4
weed/wdclient/masterclient.go

@ -18,7 +18,7 @@ type MasterClient struct {
clientType string clientType string
clientHost pb.ServerAddress clientHost pb.ServerAddress
currentMaster pb.ServerAddress currentMaster pb.ServerAddress
masters []pb.ServerAddress
masters map[string]pb.ServerAddress
grpcDialOption grpc.DialOption grpcDialOption grpc.DialOption
vidMap vidMap
@ -26,7 +26,7 @@ type MasterClient struct {
OnPeerUpdate func(update *master_pb.ClusterNodeUpdate) OnPeerUpdate func(update *master_pb.ClusterNodeUpdate)
} }
func NewMasterClient(grpcDialOption grpc.DialOption, clientType string, clientHost pb.ServerAddress, clientDataCenter string, masters []pb.ServerAddress) *MasterClient {
func NewMasterClient(grpcDialOption grpc.DialOption, clientType string, clientHost pb.ServerAddress, clientDataCenter string, masters map[string]pb.ServerAddress) *MasterClient {
return &MasterClient{ return &MasterClient{
clientType: clientType, clientType: clientType,
clientHost: clientHost, clientHost: clientHost,

3
weed/weed.go

@ -4,6 +4,7 @@ import (
"embed" "embed"
"fmt" "fmt"
weed_server "github.com/chrislusf/seaweedfs/weed/server" weed_server "github.com/chrislusf/seaweedfs/weed/server"
"github.com/chrislusf/seaweedfs/weed/util"
flag "github.com/chrislusf/seaweedfs/weed/util/fla9" flag "github.com/chrislusf/seaweedfs/weed/util/fla9"
"io" "io"
"io/fs" "io/fs"
@ -40,6 +41,8 @@ var static embed.FS
func init() { func init() {
weed_server.StaticFS, _ = fs.Sub(static, "static") weed_server.StaticFS, _ = fs.Sub(static, "static")
flag.Var(&util.ConfigurationFileDirectory, "config_dir", "directory with toml configuration files")
} }
func main() { func main() {

Loading…
Cancel
Save